From e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Tue, 29 Nov 2022 11:30:46 -0300
Subject: [PATCH 01/15] adding black formatter to all the code

---
 conftest.py                                   |   21 +-
 docs/_ext/scrapydocs.py                       |   76 +-
 docs/conf.py                                  |  175 +-
 docs/conftest.py                              |    8 +-
 docs/utils/linkfix.py                         |    4 +-
 extras/qps-bench-server.py                    |   15 +-
 extras/qpsclient.py                           |   10 +-
 scrapy/__init__.py                            |   15 +-
 scrapy/__main__.py                            |    2 +-
 scrapy/cmdline.py                             |   32 +-
 scrapy/commands/__init__.py                   |  123 +-
 scrapy/commands/bench.py                      |   22 +-
 scrapy/commands/check.py                      |   27 +-
 scrapy/commands/crawl.py                      |   11 +-
 scrapy/commands/edit.py                       |   12 +-
 scrapy/commands/fetch.py                      |   39 +-
 scrapy/commands/genspider.py                  |   94 +-
 scrapy/commands/list.py                       |    2 +-
 scrapy/commands/parse.py                      |  153 +-
 scrapy/commands/runspider.py                  |    4 +-
 scrapy/commands/settings.py                   |   40 +-
 scrapy/commands/shell.py                      |   42 +-
 scrapy/commands/startproject.py               |   61 +-
 scrapy/commands/version.py                    |   12 +-
 scrapy/commands/view.py                       |    7 +-
 scrapy/contracts/__init__.py                  |   35 +-
 scrapy/contracts/default.py                   |   60 +-
 scrapy/core/downloader/__init__.py            |   58 +-
 scrapy/core/downloader/contextfactory.py      |   49 +-
 scrapy/core/downloader/handlers/__init__.py   |   23 +-
 scrapy/core/downloader/handlers/datauri.py    |    7 +-
 scrapy/core/downloader/handlers/ftp.py        |   22 +-
 scrapy/core/downloader/handlers/http10.py     |    9 +-
 scrapy/core/downloader/handlers/http11.py     |  259 ++-
 scrapy/core/downloader/handlers/http2.py      |   38 +-
 scrapy/core/downloader/handlers/s3.py         |   60 +-
 scrapy/core/downloader/middleware.py          |   43 +-
 scrapy/core/downloader/tls.py                 |   56 +-
 scrapy/core/downloader/webclient.py           |   56 +-
 scrapy/core/engine.py                         |  154 +-
 scrapy/core/http2/agent.py                    |   14 +-
 scrapy/core/http2/protocol.py                 |  118 +-
 scrapy/core/http2/stream.py                   |  234 +-
 scrapy/core/scheduler.py                      |   96 +-
 scrapy/core/scraper.py                        |  180 +-
 scrapy/core/spidermw.py                       |  189 +-
 scrapy/crawler.py                             |   75 +-
 scrapy/downloadermiddlewares/ajaxcrawl.py     |   32 +-
 scrapy/downloadermiddlewares/cookies.py       |   35 +-
 scrapy/downloadermiddlewares/decompression.py |   21 +-
 .../downloadermiddlewares/defaultheaders.py   |    3 +-
 .../downloadermiddlewares/downloadtimeout.py  |    7 +-
 scrapy/downloadermiddlewares/httpauth.py      |   24 +-
 scrapy/downloadermiddlewares/httpcache.py     |   82 +-
 .../downloadermiddlewares/httpcompression.py  |   40 +-
 scrapy/downloadermiddlewares/httpproxy.py     |   56 +-
 scrapy/downloadermiddlewares/redirect.py      |   82 +-
 scrapy/downloadermiddlewares/retry.py         |   69 +-
 scrapy/downloadermiddlewares/robotstxt.py     |   46 +-
 scrapy/downloadermiddlewares/stats.py         |   36 +-
 scrapy/downloadermiddlewares/useragent.py     |    8 +-
 scrapy/dupefilters.py                         |   30 +-
 scrapy/exceptions.py                          |   11 +-
 scrapy/exporters.py                           |   94 +-
 scrapy/extension.py                           |    4 +-
 scrapy/extensions/closespider.py              |   47 +-
 scrapy/extensions/corestats.py                |   19 +-
 scrapy/extensions/debug.py                    |   22 +-
 scrapy/extensions/feedexport.py               |  295 +--
 scrapy/extensions/httpcache.py                |  157 +-
 scrapy/extensions/logstats.py                 |   22 +-
 scrapy/extensions/memdebug.py                 |   11 +-
 scrapy/extensions/memusage.py                 |   56 +-
 scrapy/extensions/postprocessing.py           |   27 +-
 scrapy/extensions/spiderstate.py              |    6 +-
 scrapy/extensions/statsmailer.py              |    1 -
 scrapy/extensions/telnet.py                   |   69 +-
 scrapy/extensions/throttle.py                 |   34 +-
 scrapy/http/common.py                         |    1 +
 scrapy/http/cookies.py                        |   27 +-
 scrapy/http/headers.py                        |   17 +-
 scrapy/http/request/__init__.py               |   53 +-
 scrapy/http/request/form.py                   |  101 +-
 scrapy/http/request/json_request.py           |   30 +-
 scrapy/http/request/rpc.py                    |   13 +-
 scrapy/http/response/__init__.py              |   67 +-
 scrapy/http/response/text.py                  |   84 +-
 scrapy/interfaces.py                          |    1 -
 scrapy/item.py                                |   19 +-
 scrapy/link.py                                |   12 +-
 scrapy/linkextractors/__init__.py             |   86 +-
 scrapy/linkextractors/lxmlhtml.py             |   85 +-
 scrapy/loader/common.py                       |    2 +-
 scrapy/loader/processors.py                   |   12 +-
 scrapy/logformatter.py                        |   80 +-
 scrapy/mail.py                                |  123 +-
 scrapy/middleware.py                          |   40 +-
 scrapy/pipelines/__init__.py                  |   12 +-
 scrapy/pipelines/files.py                     |  321 +--
 scrapy/pipelines/images.py                    |  150 +-
 scrapy/pipelines/media.py                     |   69 +-
 scrapy/pqueues.py                             |   37 +-
 scrapy/resolver.py                            |   24 +-
 scrapy/responsetypes.py                       |   65 +-
 scrapy/robotstxt.py                           |   16 +-
 scrapy/selector/unified.py                    |   19 +-
 scrapy/settings/__init__.py                   |   57 +-
 scrapy/settings/default_settings.py           |  220 +-
 scrapy/shell.py                               |   63 +-
 scrapy/signalmanager.py                       |   11 +-
 scrapy/spiderloader.py                        |   19 +-
 scrapy/spidermiddlewares/depth.py             |   27 +-
 scrapy/spidermiddlewares/httperror.py         |   24 +-
 scrapy/spidermiddlewares/offsite.py           |   29 +-
 scrapy/spidermiddlewares/referer.py           |   92 +-
 scrapy/spidermiddlewares/urllength.py         |   11 +-
 scrapy/spiders/__init__.py                    |   19 +-
 scrapy/spiders/crawl.py                       |   20 +-
 scrapy/spiders/feed.py                        |   46 +-
 scrapy/spiders/init.py                        |    2 +-
 scrapy/spiders/sitemap.py                     |   25 +-
 scrapy/squeues.py                             |   31 +-
 scrapy/statscollectors.py                     |   11 +-
 scrapy/utils/asyncgen.py                      |    2 +-
 scrapy/utils/benchserver.py                   |   17 +-
 scrapy/utils/boto.py                          |    1 +
 scrapy/utils/conf.py                          |   82 +-
 scrapy/utils/console.py                       |   41 +-
 scrapy/utils/curl.py                          |   50 +-
 scrapy/utils/datatypes.py                     |    2 +
 scrapy/utils/decorators.py                    |    5 +
 scrapy/utils/defer.py                         |   53 +-
 scrapy/utils/deprecate.py                     |   36 +-
 scrapy/utils/display.py                       |    3 +-
 scrapy/utils/ftp.py                           |    8 +-
 scrapy/utils/gz.py                            |   10 +-
 scrapy/utils/httpobj.py                       |    4 +-
 scrapy/utils/iterators.py                     |   67 +-
 scrapy/utils/job.py                           |    2 +-
 scrapy/utils/log.py                           |   91 +-
 scrapy/utils/misc.py                          |   54 +-
 scrapy/utils/ossignal.py                      |    5 +-
 scrapy/utils/project.py                       |   41 +-
 scrapy/utils/python.py                        |   43 +-
 scrapy/utils/reactor.py                       |   15 +-
 scrapy/utils/reqser.py                        |   14 +-
 scrapy/utils/request.py                       |  156 +-
 scrapy/utils/response.py                      |   37 +-
 scrapy/utils/signal.py                        |   59 +-
 scrapy/utils/sitemap.py                       |   22 +-
 scrapy/utils/spider.py                        |   20 +-
 scrapy/utils/ssl.py                           |   32 +-
 scrapy/utils/template.py                      |   14 +-
 scrapy/utils/test.py                          |   27 +-
 scrapy/utils/testproc.py                      |   10 +-
 scrapy/utils/testsite.py                      |   23 +-
 scrapy/utils/trackref.py                      |    3 +-
 scrapy/utils/url.py                           |   60 +-
 setup.py                                      |  110 +-
 tests/CrawlerProcess/asyncio_custom_loop.py   |   12 +-
 .../CrawlerProcess/asyncio_deferred_signal.py |   10 +-
 .../asyncio_enabled_no_reactor.py             |   10 +-
 .../CrawlerProcess/asyncio_enabled_reactor.py |   11 +-
 .../asyncio_enabled_reactor_different_loop.py |   13 +-
 .../asyncio_enabled_reactor_same_loop.py      |   13 +-
 .../caching_hostname_resolver.py              |   15 +-
 .../caching_hostname_resolver_ipv6.py         |   11 +-
 tests/CrawlerProcess/default_name_resolver.py |    1 +
 tests/CrawlerProcess/multi.py                 |    2 +-
 tests/CrawlerProcess/reactor_default.py       |    2 +-
 .../reactor_default_twisted_reactor_select.py |   10 +-
 tests/CrawlerProcess/reactor_select.py        |    2 +-
 ..._select_subclass_twisted_reactor_select.py |   10 +-
 .../reactor_select_twisted_reactor_select.py  |   10 +-
 tests/CrawlerProcess/simple.py                |    2 +-
 .../CrawlerProcess/twisted_reactor_asyncio.py |   10 +-
 .../twisted_reactor_custom_settings.py        |    2 +-
 ...wisted_reactor_custom_settings_conflict.py |    4 +-
 .../twisted_reactor_custom_settings_same.py   |    4 +-
 tests/CrawlerProcess/twisted_reactor_poll.py  |   10 +-
 .../CrawlerProcess/twisted_reactor_select.py  |   10 +-
 tests/CrawlerRunner/ip_address.py             |    2 +-
 tests/__init__.py                             |   16 +-
 tests/ftpserver.py                            |    8 +-
 tests/keys/__init__.py                        |    4 +-
 tests/mocks/dummydbm.py                       |    3 +-
 tests/mockserver.py                           |   90 +-
 tests/pipelines.py                            |    2 -
 tests/spiders.py                              |  175 +-
 tests/test_closespider.py                     |   33 +-
 tests/test_cmdline/__init__.py                |   39 +-
 tests/test_cmdline/extensions.py              |    3 +-
 tests/test_cmdline/settings.py                |   10 +-
 .../__init__.py                               |    7 +-
 .../test_spider/pipelines.py                  |    4 +-
 .../test_spider/settings.py                   |    4 +-
 .../test_spider/spiders/exception.py          |    6 +-
 .../test_spider/spiders/normal.py             |    6 +-
 tests/test_command_check.py                   |   21 +-
 tests/test_command_fetch.py                   |   26 +-
 tests/test_command_parse.py                   |  179 +-
 tests/test_command_shell.py                   |   91 +-
 tests/test_command_version.py                 |   28 +-
 tests/test_commands.py                        |  594 +++---
 tests/test_contracts.py                       |  122 +-
 tests/test_core_downloader.py                 |    5 +-
 tests/test_crawl.py                           |  236 ++-
 tests/test_crawler.py                         |  346 +--
 tests/test_dependencies.py                    |   22 +-
 tests/test_downloader_handlers.py             |  659 +++---
 tests/test_downloader_handlers_http2.py       |  111 +-
 tests/test_downloadermiddleware.py            |   82 +-
 ...test_downloadermiddleware_ajaxcrawlable.py |   32 +-
 tests/test_downloadermiddleware_cookies.py    |  475 +++--
 ...test_downloadermiddleware_decompression.py |   18 +-
 ...est_downloadermiddleware_defaultheaders.py |   13 +-
 ...st_downloadermiddleware_downloadtimeout.py |   17 +-
 tests/test_downloadermiddleware_httpauth.py   |   78 +-
 tests/test_downloadermiddleware_httpcache.py  |  358 ++--
 ...st_downloadermiddleware_httpcompression.py |  279 +--
 tests/test_downloadermiddleware_httpproxy.py  |  383 ++--
 tests/test_downloadermiddleware_redirect.py   |  324 +--
 tests/test_downloadermiddleware_retry.py      |  194 +-
 tests/test_downloadermiddleware_robotstxt.py  |  153 +-
 tests/test_downloadermiddleware_stats.py      |   40 +-
 tests/test_downloadermiddleware_useragent.py  |   40 +-
 tests/test_dupefilters.py                     |  178 +-
 tests/test_engine.py                          |  192 +-
 tests/test_engine_stop_download_bytes.py      |   43 +-
 tests/test_engine_stop_download_headers.py    |   47 +-
 tests/test_exporters.py                       |  243 ++-
 tests/test_extension_telnet.py                |   11 +-
 tests/test_feedexport.py                      | 1884 +++++++++--------
 tests/test_http2_client_protocol.py           |  360 ++--
 tests/test_http_cookies.py                    |   33 +-
 tests/test_http_headers.py                    |  185 +-
 tests/test_http_request.py                    | 1056 +++++----
 tests/test_http_response.py                   |  649 +++---
 tests/test_item.py                            |  193 +-
 tests/test_link.py                            |   21 +-
 tests/test_linkextractors.py                  |  763 +++++--
 tests/test_loader.py                          |  400 ++--
 tests/test_loader_deprecated.py               |  497 +++--
 tests/test_logformatter.py                    |  109 +-
 tests/test_mail.py                            |  144 +-
 tests/test_middleware.py                      |   32 +-
 tests/test_pipeline_crawl.py                  |   90 +-
 tests/test_pipeline_files.py                  |  353 +--
 tests/test_pipeline_images.py                 |  320 ++-
 tests/test_pipeline_media.py                  |  304 +--
 tests/test_pipelines.py                       |   23 +-
 tests/test_pqueues.py                         |   26 +-
 tests/test_proxy_connect.py                   |   59 +-
 tests/test_request_attribute_binding.py       |   76 +-
 tests/test_request_cb_kwargs.py               |  134 +-
 tests/test_request_dict.py                    |   85 +-
 tests/test_request_left.py                    |   11 +-
 tests/test_responsetypes.py                   |   97 +-
 tests/test_robotstxt_interface.py             |   98 +-
 tests/test_scheduler.py                       |  105 +-
 tests/test_scheduler_base.py                  |    6 +-
 tests/test_selector.py                        |   95 +-
 tests/test_settings/__init__.py               |  477 +++--
 tests/test_settings/default_settings.py       |    5 +-
 tests/test_signals.py                         |   11 +-
 tests/test_spider.py                          |  436 ++--
 tests/test_spiderloader/__init__.py           |  120 +-
 .../test_spiders/nested/spider4.py            |    4 +-
 .../test_spiderloader/test_spiders/spider3.py |    4 +-
 tests/test_spidermiddleware.py                |  231 +-
 tests/test_spidermiddleware_depth.py          |   17 +-
 tests/test_spidermiddleware_httperror.py      |  112 +-
 tests/test_spidermiddleware_offsite.py        |   69 +-
 tests/test_spidermiddleware_output_chain.py   |  217 +-
 tests/test_spidermiddleware_referer.py        | 1161 ++++++----
 tests/test_spidermiddleware_urllength.py      |   21 +-
 tests/test_spiderstate.py                     |   13 +-
 tests/test_squeues.py                         |   44 +-
 tests/test_squeues_request.py                 |   23 +-
 tests/test_stats.py                           |   94 +-
 tests/test_toplevel.py                        |    5 +-
 tests/test_urlparse_monkeypatches.py          |   11 +-
 tests/test_utils_asyncio.py                   |    7 +-
 tests/test_utils_conf.py                      |  242 ++-
 tests/test_utils_console.py                   |   20 +-
 tests/test_utils_curl.py                      |   53 +-
 tests/test_utils_datatypes.py                 |  161 +-
 tests/test_utils_defer.py                     |   24 +-
 tests/test_utils_deprecate.py                 |  132 +-
 tests/test_utils_display.py                   |   37 +-
 tests/test_utils_gz.py                        |   31 +-
 tests/test_utils_httpobj.py                   |    1 -
 tests/test_utils_iterators.py                 |  339 +--
 tests/test_utils_log.py                       |   70 +-
 tests/test_utils_misc/__init__.py             |  145 +-
 ...t_return_with_argument_inside_generator.py |   28 +-
 tests/test_utils_project.py                   |   37 +-
 tests/test_utils_python.py                    |  105 +-
 tests/test_utils_request.py                   |  311 ++-
 tests/test_utils_response.py                  |  140 +-
 tests/test_utils_serialize.py                 |   31 +-
 tests/test_utils_signal.py                    |   26 +-
 tests/test_utils_sitemap.py                   |  170 +-
 tests/test_utils_spider.py                    |    8 +-
 tests/test_utils_template.py                  |   19 +-
 tests/test_utils_trackref.py                  |   34 +-
 tests/test_utils_url.py                       |  642 +++---
 tests/test_webclient.py                       |  306 +--
 308 files changed, 16487 insertions(+), 11921 deletions(-)

diff --git a/conftest.py b/conftest.py
index 2a5d55083..585356a3e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -9,7 +9,7 @@ from tests.keys import generate_keys
 
 
 def _py_files(folder):
-    return (str(p) for p in Path(folder).rglob('*.py'))
+    return (str(p) for p in Path(folder).rglob("*.py"))
 
 
 collect_ignore = [
@@ -21,16 +21,16 @@ collect_ignore = [
     *_py_files("tests/CrawlerRunner"),
 ]
 
-with Path('tests/ignores.txt').open(encoding="utf-8") as reader:
+with Path("tests/ignores.txt").open(encoding="utf-8") as reader:
     for line in reader:
         file_path = line.strip()
-        if file_path and file_path[0] != '#':
+        if file_path and file_path[0] != "#":
             collect_ignore.append(file_path)
 
 if not H2_ENABLED:
     collect_ignore.extend(
         (
-            'scrapy/core/downloader/handlers/http2.py',
+            "scrapy/core/downloader/handlers/http2.py",
             *_py_files("scrapy/core/http2"),
         )
     )
@@ -50,7 +50,7 @@ def pytest_addoption(parser):
     )
 
 
-@pytest.fixture(scope='class')
+@pytest.fixture(scope="class")
 def reactor_pytest(request):
     if not request.cls:
         # doctests
@@ -61,14 +61,17 @@ def reactor_pytest(request):
 
 @pytest.fixture(autouse=True)
 def only_asyncio(request, reactor_pytest):
-    if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
-        pytest.skip('This test is only run with --reactor=asyncio')
+    if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
+        pytest.skip("This test is only run with --reactor=asyncio")
 
 
 @pytest.fixture(autouse=True)
 def only_not_asyncio(request, reactor_pytest):
-    if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
-        pytest.skip('This test is only run without --reactor=asyncio')
+    if (
+        request.node.get_closest_marker("only_not_asyncio")
+        and reactor_pytest == "asyncio"
+    ):
+        pytest.skip("This test is only run without --reactor=asyncio")
 
 
 def pytest_configure(config):
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 337604cf1..1419792fc 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -11,15 +11,15 @@ class settingslist_node(nodes.General, nodes.Element):
 
 class SettingsListDirective(Directive):
     def run(self):
-        return [settingslist_node('')]
+        return [settingslist_node("")]
 
 
 def is_setting_index(node):
-    if node.tagname == 'index' and node['entries']:
+    if node.tagname == "index" and node["entries"]:
         # index entries for setting directives look like:
         # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
-        entry_type, info, refid = node['entries'][0][:3]
-        return entry_type == 'pair' and info.endswith('; setting')
+        entry_type, info, refid = node["entries"][0][:3]
+        return entry_type == "pair" and info.endswith("; setting")
     return False
 
 
@@ -30,14 +30,14 @@ def get_setting_target(node):
 
 def get_setting_name_and_refid(node):
     """Extract setting name from directive index node"""
-    entry_type, info, refid = node['entries'][0][:3]
-    return info.replace('; setting', ''), refid
+    entry_type, info, refid = node["entries"][0][:3]
+    return info.replace("; setting", ""), refid
 
 
 def collect_scrapy_settings_refs(app, doctree):
     env = app.builder.env
 
-    if not hasattr(env, 'scrapy_all_settings'):
+    if not hasattr(env, "scrapy_all_settings"):
         env.scrapy_all_settings = []
 
     for node in doctree.traverse(is_setting_index):
@@ -46,18 +46,23 @@ def collect_scrapy_settings_refs(app, doctree):
 
         setting_name, refid = get_setting_name_and_refid(node)
 
-        env.scrapy_all_settings.append({
-            'docname': env.docname,
-            'setting_name': setting_name,
-            'refid': refid,
-        })
+        env.scrapy_all_settings.append(
+            {
+                "docname": env.docname,
+                "setting_name": setting_name,
+                "refid": refid,
+            }
+        )
 
 
 def make_setting_element(setting_data, app, fromdocname):
-    refnode = make_refnode(app.builder, fromdocname,
-                           todocname=setting_data['docname'],
-                           targetid=setting_data['refid'],
-                           child=nodes.Text(setting_data['setting_name']))
+    refnode = make_refnode(
+        app.builder,
+        fromdocname,
+        todocname=setting_data["docname"],
+        targetid=setting_data["refid"],
+        child=nodes.Text(setting_data["setting_name"]),
+    )
     p = nodes.paragraph()
     p += refnode
 
@@ -71,10 +76,13 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
 
     for node in doctree.traverse(settingslist_node):
         settings_list = nodes.bullet_list()
-        settings_list.extend([make_setting_element(d, app, fromdocname)
-                              for d in sorted(env.scrapy_all_settings,
-                                              key=itemgetter('setting_name'))
-                              if fromdocname != d['docname']])
+        settings_list.extend(
+            [
+                make_setting_element(d, app, fromdocname)
+                for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name"))
+                if fromdocname != d["docname"]
+            ]
+        )
         node.replace_self(settings_list)
 
 
@@ -99,41 +107,41 @@ def setup(app):
         rolename="reqmeta",
         indextemplate="pair: %s; reqmeta",
     )
-    app.add_role('source', source_role)
-    app.add_role('commit', commit_role)
-    app.add_role('issue', issue_role)
-    app.add_role('rev', rev_role)
+    app.add_role("source", source_role)
+    app.add_role("commit", commit_role)
+    app.add_role("issue", issue_role)
+    app.add_role("rev", rev_role)
 
     app.add_node(settingslist_node)
-    app.add_directive('settingslist', SettingsListDirective)
+    app.add_directive("settingslist", SettingsListDirective)
 
-    app.connect('doctree-read', collect_scrapy_settings_refs)
-    app.connect('doctree-resolved', replace_settingslist_nodes)
+    app.connect("doctree-read", collect_scrapy_settings_refs)
+    app.connect("doctree-resolved", replace_settingslist_nodes)
 
 
 def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
-    ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
+    ref = "https://github.com/scrapy/scrapy/blob/master/" + text
     set_classes(options)
     node = nodes.reference(rawtext, text, refuri=ref, **options)
     return [node], []
 
 
 def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
-    ref = 'https://github.com/scrapy/scrapy/issues/' + text
+    ref = "https://github.com/scrapy/scrapy/issues/" + text
     set_classes(options)
-    node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
+    node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options)
     return [node], []
 
 
 def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
-    ref = 'https://github.com/scrapy/scrapy/commit/' + text
+    ref = "https://github.com/scrapy/scrapy/commit/" + text
     set_classes(options)
-    node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
+    node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options)
     return [node], []
 
 
 def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
-    ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
+    ref = "http://hg.scrapy.org/scrapy/changeset/" + text
     set_classes(options)
-    node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)
+    node = nodes.reference(rawtext, "r" + text, refuri=ref, **options)
     return [node], []
diff --git a/docs/conf.py b/docs/conf.py
index d2a77003e..38ca81932 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -25,30 +25,30 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    'hoverxref.extension',
-    'notfound.extension',
-    'scrapydocs',
-    'sphinx.ext.autodoc',
-    'sphinx.ext.coverage',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.viewcode',
+    "hoverxref.extension",
+    "notfound.extension",
+    "scrapydocs",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.coverage",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.viewcode",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8'
+# source_encoding = 'utf-8'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'Scrapy'
-copyright = f'2008–{datetime.now().year}, Scrapy developers'
+project = "Scrapy"
+copyright = f"2008–{datetime.now().year}, Scrapy developers"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -57,50 +57,51 @@ copyright = f'2008–{datetime.now().year}, Scrapy developers'
 # The short X.Y version.
 try:
     import scrapy
-    version = '.'.join(map(str, scrapy.version_info[:2]))
+
+    version = ".".join(map(str, scrapy.version_info[:2]))
     release = scrapy.__version__
 except ImportError:
-    version = ''
-    release = ''
+    version = ""
+    release = ""
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-language = 'en'
+language = "en"
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
-#unused_docs = []
+# unused_docs = []
 
-exclude_patterns = ['build']
+exclude_patterns = ["build"]
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_trees = ['.build']
+exclude_trees = [".build"]
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # List of Sphinx warnings that will not be raised
-suppress_warnings = ['epub.unknown_project_files']
+suppress_warnings = ["epub.unknown_project_files"]
 
 
 # Options for HTML output
@@ -108,17 +109,18 @@ suppress_warnings = ['epub.unknown_project_files']
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # Add path to the RTD explicitly to robustify builds (otherwise might
 # fail in a clean Debian build env)
 import sphinx_rtd_theme
+
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
@@ -128,44 +130,44 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = "%b %d, %Y"
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_use_modindex = True
+# html_use_modindex = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, the reST sources are included in the HTML build as _sources/<name>.
 html_copy_source = True
@@ -173,16 +175,16 @@ html_copy_source = True
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = ''
+# html_file_suffix = ''
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'Scrapydoc'
+htmlhelp_basename = "Scrapydoc"
 
 html_css_files = [
-    'custom.css',
+    "custom.css",
 ]
 
 
@@ -190,34 +192,33 @@ html_css_files = [
 # ------------------------
 
 # The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
+# latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
+# latex_font_size = '10pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 latex_documents = [
-  ('index', 'Scrapy.tex', 'Scrapy Documentation',
-   'Scrapy developers', 'manual'),
+    ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
+# latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_use_modindex = True
+# latex_use_modindex = True
 
 
 # Options for the linkcheck builder
@@ -226,8 +227,9 @@ latex_documents = [
 # A list of regular expressions that match URIs that should not be checked when
 # doing a linkcheck build.
 linkcheck_ignore = [
-    'http://localhost:\d+', 'http://hg.scrapy.org',
-    'http://directory.google.com/'
+    "http://localhost:\d+",
+    "http://hg.scrapy.org",
+    "http://directory.google.com/",
 ]
 
 
@@ -237,44 +239,35 @@ coverage_ignore_pyobjects = [
     # Contract’s add_pre_hook and add_post_hook are not documented because
     # they should be transparent to contract developers, for whom pre_hook and
     # post_hook should be the actual concern.
-    r'\bContract\.add_(pre|post)_hook$',
-
+    r"\bContract\.add_(pre|post)_hook$",
     # ContractsManager is an internal class, developers are not expected to
     # interact with it directly in any way.
-    r'\bContractsManager\b$',
-
+    r"\bContractsManager\b$",
     # For default contracts we only want to document their general purpose in
     # their __init__ method, the methods they reimplement to achieve that purpose
     # should be irrelevant to developers using those contracts.
-    r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
-
+    r"\w+Contract\.(adjust_request_args|(pre|post)_process)$",
     # Methods of downloader middlewares are not documented, only the classes
     # themselves, since downloader middlewares are controlled through Scrapy
     # settings.
-    r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
-
+    r"^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.",
     # Base classes of downloader middlewares are implementation details that
     # are not meant for users.
-    r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
-
+    r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware",
     # Private exception used by the command-line interface implementation.
-    r'^scrapy\.exceptions\.UsageError',
-
+    r"^scrapy\.exceptions\.UsageError",
     # Methods of BaseItemExporter subclasses are only documented in
     # BaseItemExporter.
-    r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
-
+    r"^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.",
     # Extension behavior is only modified through settings. Methods of
     # extension classes, as well as helper functions, are implementation
     # details that are not documented.
-    r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.',  # methods
-    r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]',  # helper functions
-
+    r"^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.",  # methods
+    r"^scrapy\.extensions\.[a-z]\w*?\.[a-z]",  # helper functions
     # Never documented before, and deprecated now.
-    r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
-
+    r"^scrapy\.linkextractors\.FilteringLinkExtractor$",
     # Implementation detail of LxmlLinkExtractor
-    r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
+    r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
 ]
 
 
@@ -282,18 +275,18 @@ coverage_ignore_pyobjects = [
 # -------------------------------------
 
 intersphinx_mapping = {
-    'attrs': ('https://www.attrs.org/en/stable/', None),
-    'coverage': ('https://coverage.readthedocs.io/en/stable', None),
-    'cryptography' : ('https://cryptography.io/en/latest/', None),
-    'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
-    'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
-    'pytest': ('https://docs.pytest.org/en/latest', None),
-    'python': ('https://docs.python.org/3', None),
-    'sphinx': ('https://www.sphinx-doc.org/en/master', None),
-    'tox': ('https://tox.wiki/en/latest/', None),
-    'twisted': ('https://docs.twisted.org/en/stable/', None),
-    'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
-    'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
+    "attrs": ("https://www.attrs.org/en/stable/", None),
+    "coverage": ("https://coverage.readthedocs.io/en/stable", None),
+    "cryptography": ("https://cryptography.io/en/latest/", None),
+    "cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
+    "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
+    "pytest": ("https://docs.pytest.org/en/latest", None),
+    "python": ("https://docs.python.org/3", None),
+    "sphinx": ("https://www.sphinx-doc.org/en/master", None),
+    "tox": ("https://tox.wiki/en/latest/", None),
+    "twisted": ("https://docs.twisted.org/en/stable/", None),
+    "twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
+    "w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
 }
 intersphinx_disabled_reftypes = []
 
@@ -313,16 +306,16 @@ hoverxref_role_types = {
     "setting": "tooltip",
     "signal": "tooltip",
 }
-hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
+hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
 
 
 def setup(app):
-    app.connect('autodoc-skip-member', maybe_skip_member)
+    app.connect("autodoc-skip-member", maybe_skip_member)
 
 
 def maybe_skip_member(app, what, name, obj, skip, options):
     if not skip:
         # autodocs was generating a text "alias of" for the following members
         # https://github.com/sphinx-doc/sphinx/issues/4422
-        return name in {'default_item_class', 'default_selector_class'}
+        return name in {"default_item_class", "default_selector_class"}
     return skip
diff --git a/docs/conftest.py b/docs/conftest.py
index a6dacd265..32f849a36 100644
--- a/docs/conftest.py
+++ b/docs/conftest.py
@@ -15,20 +15,20 @@ from scrapy.http.response.html import HtmlResponse
 
 
 def load_response(url: str, filename: str) -> HtmlResponse:
-    input_path = Path(__file__).parent / '_tests' / filename
+    input_path = Path(__file__).parent / "_tests" / filename
     return HtmlResponse(url, body=input_path.read_bytes())
 
 
 def setup(namespace):
-    namespace['load_response'] = load_response
+    namespace["load_response"] = load_response
 
 
 pytest_collect_file = Sybil(
     parsers=[
         DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
-        PythonCodeBlockParser(future_imports=['print_function']),
+        PythonCodeBlockParser(future_imports=["print_function"]),
         skip,
     ],
-    pattern='*.rst',
+    pattern="*.rst",
     setup=setup,
 ).pytest()
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index a0d9a1cbd..efb4c202e 100644
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -25,7 +25,7 @@ def main():
     _contents = None
 
     # A regex that matches standard linkcheck output lines
-    line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+    line_re = re.compile(r"(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))")
 
     # Read lines from the linkcheck output file
     try:
@@ -66,5 +66,5 @@ def main():
             print("Not Understood: " + line)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index a6472b1ba..622164c75 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -7,7 +7,6 @@ from twisted.internet import reactor
 
 
 class Root(Resource):
-
     def __init__(self):
         Resource.__init__(self)
         self.concurrent = 0
@@ -26,9 +25,9 @@ class Root(Resource):
         delta = now - self.lasttime
 
         # reset stats on high iter-request times caused by client restarts
-        if delta > 3: # seconds
+        if delta > 3:  # seconds
             self._reset_stats()
-            return ''
+            return ""
 
         self.tail.appendleft(delta)
         self.lasttime = now
@@ -37,15 +36,17 @@ class Root(Resource):
         if now - self.lastmark >= 3:
             self.lastmark = now
             qps = len(self.tail) / sum(self.tail)
-            print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
+            print(
+                f"samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}"
+            )
 
-        if 'latency' in request.args:
-            latency = float(request.args['latency'][0])
+        if "latency" in request.args:
+            latency = float(request.args["latency"][0])
             reactor.callLater(latency, self._finish, request)
             return NOT_DONE_YET
 
         self.concurrent -= 1
-        return ''
+        return ""
 
     def _finish(self, request):
         self.concurrent -= 1
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 28703650d..83bb08561 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -13,13 +13,13 @@ from scrapy.http import Request
 
 class QPSSpider(Spider):
 
-    name = 'qps'
-    benchurl = 'http://localhost:8880/'
+    name = "qps"
+    benchurl = "http://localhost:8880/"
 
     # Max concurrency is limited by global CONCURRENT_REQUESTS setting
     max_concurrent_requests = 8
     # Requests per second goal
-    qps = None # same as: 1 / download_delay
+    qps = None  # same as: 1 / download_delay
     download_delay = None
     # time in seconds to delay server responses
     latency = None
@@ -37,11 +37,11 @@ class QPSSpider(Spider):
     def start_requests(self):
         url = self.benchurl
         if self.latency is not None:
-            url += f'?latency={self.latency}'
+            url += f"?latency={self.latency}"
 
         slots = int(self.slots)
         if slots > 1:
-            urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
+            urls = [url.replace("localhost", f"127.0.0.{x + 1}") for x in range(slots)]
         else:
             urls = [url]
 
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 86e584396..f0d85198d 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -16,14 +16,21 @@ from scrapy.item import Item, Field
 
 
 __all__ = [
-    '__version__', 'version_info', 'twisted_version', 'Spider',
-    'Request', 'FormRequest', 'Selector', 'Item', 'Field',
+    "__version__",
+    "version_info",
+    "twisted_version",
+    "Spider",
+    "Request",
+    "FormRequest",
+    "Selector",
+    "Item",
+    "Field",
 ]
 
 
 # Scrapy and Twisted versions
 __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
-version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
+version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
@@ -34,7 +41,7 @@ if sys.version_info < (3, 7):
 
 
 # Ignore noisy twisted deprecation warnings
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
 
 
 del pkgutil
diff --git a/scrapy/__main__.py b/scrapy/__main__.py
index e467e057f..697b9b1e9 100644
--- a/scrapy/__main__.py
+++ b/scrapy/__main__.py
@@ -1,4 +1,4 @@
 from scrapy.cmdline import execute
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     execute()
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 8218a51c8..1d74923a9 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -17,7 +17,7 @@ from scrapy.utils.python import garbage_collect
 class ScrapyArgumentParser(argparse.ArgumentParser):
     def _parse_optional(self, arg_string):
         # if starts with -: it means that is a parameter not a argument
-        if arg_string[:2] == '-:':
+        if arg_string[:2] == "-:":
             return None
 
         return super()._parse_optional(arg_string)
@@ -41,12 +41,12 @@ def _get_commands_from_module(module, inproject):
     d = {}
     for cmd in _iter_command_classes(module):
         if inproject or not cmd.requires_project:
-            cmdname = cmd.__module__.split('.')[-1]
+            cmdname = cmd.__module__.split(".")[-1]
             d[cmdname] = cmd()
     return d
 
 
-def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
+def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
     for entry_point in pkg_resources.iter_entry_points(group):
         obj = entry_point.load()
@@ -58,9 +58,9 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
 
 
 def _get_commands_dict(settings, inproject):
-    cmds = _get_commands_from_module('scrapy.commands', inproject)
+    cmds = _get_commands_from_module("scrapy.commands", inproject)
     cmds.update(_get_commands_from_entry_points(inproject))
-    cmds_module = settings['COMMANDS_MODULE']
+    cmds_module = settings["COMMANDS_MODULE"]
     if cmds_module:
         cmds.update(_get_commands_from_module(cmds_module, inproject))
     return cmds
@@ -69,7 +69,7 @@ def _get_commands_dict(settings, inproject):
 def _pop_command_name(argv):
     i = 0
     for arg in argv[1:]:
-        if not arg.startswith('-'):
+        if not arg.startswith("-"):
             del argv[i]
             return arg
         i += 1
@@ -124,11 +124,11 @@ def execute(argv=None, settings=None):
         settings = get_project_settings()
         # set EDITOR from environment if available
         try:
-            editor = os.environ['EDITOR']
+            editor = os.environ["EDITOR"]
         except KeyError:
             pass
         else:
-            settings['EDITOR'] = editor
+            settings["EDITOR"] = editor
 
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
@@ -141,11 +141,13 @@ def execute(argv=None, settings=None):
         sys.exit(2)
 
     cmd = cmds[cmdname]
-    parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
-                                  usage=f"scrapy {cmdname} {cmd.syntax()}",
-                                  conflict_handler='resolve',
-                                  description=cmd.long_desc())
-    settings.setdict(cmd.default_settings, priority='command')
+    parser = ScrapyArgumentParser(
+        formatter_class=ScrapyHelpFormatter,
+        usage=f"scrapy {cmdname} {cmd.syntax()}",
+        conflict_handler="resolve",
+        description=cmd.long_desc(),
+    )
+    settings.setdict(cmd.default_settings, priority="command")
     cmd.settings = settings
     cmd.add_options(parser)
     opts, args = parser.parse_known_args(args=argv[1:])
@@ -168,12 +170,12 @@ def _run_command_profiled(cmd, args, opts):
         sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
     loc = locals()
     p = cProfile.Profile()
-    p.runctx('cmd.run(args, opts)', globals(), loc)
+    p.runctx("cmd.run(args, opts)", globals(), loc)
     if opts.profile:
         p.dump_stats(opts.profile)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     try:
         execute()
     finally:
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index b9ba3335e..f37d61321 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -27,7 +27,7 @@ class ScrapyCommand:
         self.settings: Any = None  # set in scrapy.cmdline
 
     def set_crawler(self, crawler):
-        if hasattr(self, '_crawler'):
+        if hasattr(self, "_crawler"):
             raise RuntimeError("crawler already set")
         self._crawler = crawler
 
@@ -61,41 +61,58 @@ class ScrapyCommand:
         """
         Populate option parse with options available for this command
         """
-        group = parser.add_argument_group(title='Global Options')
-        group.add_argument("--logfile", metavar="FILE",
-                           help="log file. if omitted stderr will be used")
-        group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
-                           help=f"log level (default: {self.settings['LOG_LEVEL']})")
-        group.add_argument("--nolog", action="store_true",
-                           help="disable logging completely")
-        group.add_argument("--profile", metavar="FILE", default=None,
-                           help="write python cProfile stats to FILE")
-        group.add_argument("--pidfile", metavar="FILE",
-                           help="write process ID to FILE")
-        group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
-                           help="set/override setting (may be repeated)")
+        group = parser.add_argument_group(title="Global Options")
+        group.add_argument(
+            "--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
+        )
+        group.add_argument(
+            "-L",
+            "--loglevel",
+            metavar="LEVEL",
+            default=None,
+            help=f"log level (default: {self.settings['LOG_LEVEL']})",
+        )
+        group.add_argument(
+            "--nolog", action="store_true", help="disable logging completely"
+        )
+        group.add_argument(
+            "--profile",
+            metavar="FILE",
+            default=None,
+            help="write python cProfile stats to FILE",
+        )
+        group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE")
+        group.add_argument(
+            "-s",
+            "--set",
+            action="append",
+            default=[],
+            metavar="NAME=VALUE",
+            help="set/override setting (may be repeated)",
+        )
         group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
 
     def process_options(self, args, opts):
         try:
-            self.settings.setdict(arglist_to_dict(opts.set),
-                                  priority='cmdline')
+            self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
         except ValueError:
             raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
-            self.settings.set('LOG_ENABLED', True, priority='cmdline')
-            self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
+            self.settings.set("LOG_ENABLED", True, priority="cmdline")
+            self.settings.set("LOG_FILE", opts.logfile, priority="cmdline")
 
         if opts.loglevel:
-            self.settings.set('LOG_ENABLED', True, priority='cmdline')
-            self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
+            self.settings.set("LOG_ENABLED", True, priority="cmdline")
+            self.settings.set("LOG_LEVEL", opts.loglevel, priority="cmdline")
 
         if opts.nolog:
-            self.settings.set('LOG_ENABLED', False, priority='cmdline')
+            self.settings.set("LOG_ENABLED", False, priority="cmdline")
 
         if opts.pidfile:
-            Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8")
+            Path(opts.pidfile).write_text(
+                str(os.getpid()) + os.linesep, encoding="utf-8"
+            )
 
         if opts.pdb:
             failure.startDebugMode()
@@ -111,18 +128,39 @@ class BaseRunSpiderCommand(ScrapyCommand):
     """
     Common class used to share functionality between the crawl, parse and runspider commands
     """
+
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                            help="set spider argument (may be repeated)")
-        parser.add_argument("-o", "--output", metavar="FILE", action="append",
-                            help="append scraped items to the end of FILE (use - for stdout),"
-                                 " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
-        parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
-                            help="dump scraped items into FILE, overwriting any existing file,"
-                                 " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
-        parser.add_argument("-t", "--output-format", metavar="FORMAT",
-                            help="format to use for dumping items")
+        parser.add_argument(
+            "-a",
+            dest="spargs",
+            action="append",
+            default=[],
+            metavar="NAME=VALUE",
+            help="set spider argument (may be repeated)",
+        )
+        parser.add_argument(
+            "-o",
+            "--output",
+            metavar="FILE",
+            action="append",
+            help="append scraped items to the end of FILE (use - for stdout),"
+            " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)",
+        )
+        parser.add_argument(
+            "-O",
+            "--overwrite-output",
+            metavar="FILE",
+            action="append",
+            help="dump scraped items into FILE, overwriting any existing file,"
+            " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
+        )
+        parser.add_argument(
+            "-t",
+            "--output-format",
+            metavar="FORMAT",
+            help="format to use for dumping items",
+        )
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
@@ -137,16 +175,21 @@ class BaseRunSpiderCommand(ScrapyCommand):
                 opts.output_format,
                 opts.overwrite_output,
             )
-            self.settings.set('FEEDS', feeds, priority='cmdline')
+            self.settings.set("FEEDS", feeds, priority="cmdline")
 
 
 class ScrapyHelpFormatter(argparse.HelpFormatter):
     """
     Help Formatter for scrapy command line help messages.
     """
+
     def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
-        super().__init__(prog, indent_increment=indent_increment,
-                         max_help_position=max_help_position, width=width)
+        super().__init__(
+            prog,
+            indent_increment=indent_increment,
+            max_help_position=max_help_position,
+            width=width,
+        )
 
     def _join_parts(self, part_strings):
         parts = self.format_part_strings(part_strings)
@@ -157,11 +200,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
         Underline and title case command line help message headers.
         """
         if part_strings and part_strings[0].startswith("usage: "):
-            part_strings[0] = "Usage\n=====\n  " + part_strings[0][len('usage: '):]
-        headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
+            part_strings[0] = "Usage\n=====\n  " + part_strings[0][len("usage: ") :]
+        headings = [
+            i for i in range(len(part_strings)) if part_strings[i].endswith(":\n")
+        ]
         for index in headings[::-1]:
-            char = '-' if "Global Options" in part_strings[index] else '='
+            char = "-" if "Global Options" in part_strings[index] else "="
             part_strings[index] = part_strings[index][:-2].title()
-            underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
+            underline = "".join(["\n", (char * len(part_strings[index])), "\n"])
             part_strings.insert(index + 1, underline)
         return part_strings
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 6bdf9eae0..2e2a21f00 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -11,9 +11,9 @@ from scrapy.linkextractors import LinkExtractor
 class Command(ScrapyCommand):
 
     default_settings = {
-        'LOG_LEVEL': 'INFO',
-        'LOGSTATS_INTERVAL': 1,
-        'CLOSESPIDER_TIMEOUT': 10,
+        "LOG_LEVEL": "INFO",
+        "LOGSTATS_INTERVAL": 1,
+        "CLOSESPIDER_TIMEOUT": 10,
     }
 
     def short_desc(self):
@@ -26,12 +26,11 @@ class Command(ScrapyCommand):
 
 
 class _BenchServer:
-
     def __enter__(self):
         from scrapy.utils.test import get_testenv
-        pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver']
-        self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE,
-                                     env=get_testenv())
+
+        pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
+        self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
         self.proc.stdout.readline()
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -42,15 +41,16 @@ class _BenchServer:
 
 class _BenchSpider(scrapy.Spider):
     """A spider that follows all links"""
-    name = 'follow'
+
+    name = "follow"
     total = 10000
     show = 20
-    baseurl = 'http://localhost:8998'
+    baseurl = "http://localhost:8998"
     link_extractor = LinkExtractor()
 
     def start_requests(self):
-        qargs = {'total': self.total, 'show': self.show}
-        url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
+        qargs = {"total": self.total, "show": self.show}
+        url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
         return [scrapy.Request(url, dont_filter=True)]
 
     def parse(self, response):
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index a16f4beb7..efc7a46ed 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -39,7 +39,7 @@ class TextTestResult(_TextTestResult):
 
 class Command(ScrapyCommand):
     requires_project = True
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {"LOG_ENABLED": False}
 
     def syntax(self):
         return "[options] <spider>"
@@ -49,14 +49,25 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_argument("-l", "--list", dest="list", action="store_true",
-                            help="only list contracts, without checking them")
-        parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
-                            help="print contract tests for all spiders")
+        parser.add_argument(
+            "-l",
+            "--list",
+            dest="list",
+            action="store_true",
+            help="only list contracts, without checking them",
+        )
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            dest="verbose",
+            default=False,
+            action="store_true",
+            help="print contract tests for all spiders",
+        )
 
     def run(self, args, opts):
         # load contracts
-        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
+        contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
         conman = ContractsManager(load_object(c) for c in contracts)
         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
         result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
@@ -66,7 +77,7 @@ class Command(ScrapyCommand):
 
         spider_loader = self.crawler_process.spider_loader
 
-        with set_environ(SCRAPY_CHECK='true'):
+        with set_environ(SCRAPY_CHECK="true"):
             for spidername in args or spider_loader.list():
                 spidercls = spider_loader.load(spidername)
                 spidercls.start_requests = lambda s: conman.from_spider(s, result)
@@ -85,7 +96,7 @@ class Command(ScrapyCommand):
                         continue
                     print(spider)
                     for method in sorted(methods):
-                        print(f'  * {method}')
+                        print(f"  * {method}")
             else:
                 start = time.time()
                 self.crawler_process.start()
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 0f2a21b85..df8006f36 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -16,18 +16,23 @@ class Command(BaseRunSpiderCommand):
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
-            raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
+            raise UsageError(
+                "running 'scrapy crawl' with more than one spider is not supported"
+            )
         spname = args[0]
 
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
 
-        if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
+        if getattr(crawl_defer, "result", None) is not None and issubclass(
+            crawl_defer.result.type, Exception
+        ):
             self.exitcode = 1
         else:
             self.crawler_process.start()
 
             if (
                 self.crawler_process.bootstrap_failed
-                or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
+                or hasattr(self.crawler_process, "has_exception")
+                and self.crawler_process.has_exception
             ):
                 self.exitcode = 1
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 177b20143..537b2013c 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -8,7 +8,7 @@ from scrapy.exceptions import UsageError
 class Command(ScrapyCommand):
 
     requires_project = True
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {"LOG_ENABLED": False}
 
     def syntax(self):
         return "<spider>"
@@ -17,8 +17,10 @@ class Command(ScrapyCommand):
         return "Edit spider"
 
     def long_desc(self):
-        return ("Edit a spider using the editor defined in the EDITOR environment"
-                " variable or else the EDITOR setting")
+        return (
+            "Edit a spider using the editor defined in the EDITOR environment"
+            " variable or else the EDITOR setting"
+        )
 
     def _err(self, msg):
         sys.stderr.write(msg + os.linesep)
@@ -28,12 +30,12 @@ class Command(ScrapyCommand):
         if len(args) != 1:
             raise UsageError()
 
-        editor = self.settings['EDITOR']
+        editor = self.settings["EDITOR"]
         try:
             spidercls = self.crawler_process.spider_loader.load(args[0])
         except KeyError:
             return self._err(f"Spider not found: {args[0]}")
 
         sfile = sys.modules[spidercls.__module__].__file__
-        sfile = sfile.replace('.pyc', '.py')
+        sfile = sfile.replace(".pyc", ".py")
         self.exitcode = os.system(f'{editor} "{sfile}"')
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 9b2ebb37f..918db55c6 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -27,38 +27,51 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_argument("--spider", dest="spider", help="use this spider")
-        parser.add_argument("--headers", dest="headers", action="store_true",
-                            help="print response HTTP headers instead of body")
-        parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
-                            help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_argument(
+            "--headers",
+            dest="headers",
+            action="store_true",
+            help="print response HTTP headers instead of body",
+        )
+        parser.add_argument(
+            "--no-redirect",
+            dest="no_redirect",
+            action="store_true",
+            default=False,
+            help="do not handle HTTP 3xx status codes and print response as-is",
+        )
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
             for value in values:
-                self._print_bytes(prefix + b' ' + key + b': ' + value)
+                self._print_bytes(prefix + b" " + key + b": " + value)
 
     def _print_response(self, response, opts):
         if opts.headers:
-            self._print_headers(response.request.headers, b'>')
-            print('>')
-            self._print_headers(response.headers, b'<')
+            self._print_headers(response.request.headers, b">")
+            print(">")
+            self._print_headers(response.headers, b"<")
         else:
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        sys.stdout.buffer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b"\n")
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):
             raise UsageError()
-        request = Request(args[0], callback=self._print_response,
-                          cb_kwargs={"opts": opts}, dont_filter=True)
+        request = Request(
+            args[0],
+            callback=self._print_response,
+            cb_kwargs={"opts": opts},
+            dont_filter=True,
+        )
         # by default, let the framework handle redirects,
         # i.e. command handles all codes expect 3xx
         if not opts.no_redirect:
-            request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
+            request.meta["handle_httpstatus_list"] = SequenceExclude(range(300, 400))
         else:
-            request.meta['handle_httpstatus_all'] = True
+            request.meta["handle_httpstatus_all"] = True
 
         spidercls = DefaultSpider
         spider_loader = self.crawler_process.spider_loader
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index b7c2f85fb..e880e44a9 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -18,7 +18,7 @@ def sanitize_module_name(module_name):
     with underscores and prefixing it with a letter if it doesn't start
     with one
     """
-    module_name = module_name.replace('-', '_').replace('.', '_')
+    module_name = module_name.replace("-", "_").replace(".", "_")
     if module_name[0] not in string.ascii_letters:
         module_name = "a" + module_name
     return module_name
@@ -27,7 +27,7 @@ def sanitize_module_name(module_name):
 def extract_domain(url):
     """Extract domain name from URL string"""
     o = urlparse(url)
-    if o.scheme == '' and o.netloc == '':
+    if o.scheme == "" and o.netloc == "":
         o = urlparse("//" + url.lstrip("/"))
     return o.netloc
 
@@ -35,7 +35,7 @@ def extract_domain(url):
 class Command(ScrapyCommand):
 
     requires_project = False
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {"LOG_ENABLED": False}
 
     def syntax(self):
         return "[options] <name> <domain>"
@@ -45,16 +45,40 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_argument("-l", "--list", dest="list", action="store_true",
-                            help="List available templates")
-        parser.add_argument("-e", "--edit", dest="edit", action="store_true",
-                            help="Edit spider after creating it")
-        parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
-                            help="Dump template to standard output")
-        parser.add_argument("-t", "--template", dest="template", default="basic",
-                            help="Uses a custom template.")
-        parser.add_argument("--force", dest="force", action="store_true",
-                            help="If the spider already exists, overwrite it with the template")
+        parser.add_argument(
+            "-l",
+            "--list",
+            dest="list",
+            action="store_true",
+            help="List available templates",
+        )
+        parser.add_argument(
+            "-e",
+            "--edit",
+            dest="edit",
+            action="store_true",
+            help="Edit spider after creating it",
+        )
+        parser.add_argument(
+            "-d",
+            "--dump",
+            dest="dump",
+            metavar="TEMPLATE",
+            help="Dump template to standard output",
+        )
+        parser.add_argument(
+            "-t",
+            "--template",
+            dest="template",
+            default="basic",
+            help="Uses a custom template.",
+        )
+        parser.add_argument(
+            "--force",
+            dest="force",
+            action="store_true",
+            help="If the spider already exists, overwrite it with the template",
+        )
 
     def run(self, args, opts):
         if opts.list:
@@ -72,7 +96,7 @@ class Command(ScrapyCommand):
         domain = extract_domain(url)
         module = sanitize_module_name(name)
 
-        if self.settings.get('BOT_NAME') == module:
+        if self.settings.get("BOT_NAME") == module:
             print("Cannot create a spider with the same name as your project")
             return
 
@@ -87,17 +111,17 @@ class Command(ScrapyCommand):
 
     def _genspider(self, module, name, domain, template_name, template_file):
         """Generate the spider module, based on the given template"""
-        capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
+        capitalized_module = "".join(s.capitalize() for s in module.split("_"))
         tvars = {
-            'project_name': self.settings.get('BOT_NAME'),
-            'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
-            'module': module,
-            'name': name,
-            'domain': domain,
-            'classname': f'{capitalized_module}Spider'
+            "project_name": self.settings.get("BOT_NAME"),
+            "ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
+            "module": module,
+            "name": name,
+            "domain": domain,
+            "classname": f"{capitalized_module}Spider",
         }
-        if self.settings.get('NEWSPIDER_MODULE'):
-            spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+        if self.settings.get("NEWSPIDER_MODULE"):
+            spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
             spiders_dir = Path(spiders_module.__file__).parent.resolve()
         else:
             spiders_module = None
@@ -105,13 +129,15 @@ class Command(ScrapyCommand):
         spider_file = f"{spiders_dir / module}.py"
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print(f"Created spider {name!r} using template {template_name!r} ",
-              end=('' if spiders_module else '\n'))
+        print(
+            f"Created spider {name!r} using template {template_name!r} ",
+            end=("" if spiders_module else "\n"),
+        )
         if spiders_module:
             print(f"in module:\n  {spiders_module.__name__}.{module}")
 
     def _find_template(self, template: str) -> Optional[Path]:
-        template_file = Path(self.templates_dir, f'{template}.tmpl')
+        template_file = Path(self.templates_dir, f"{template}.tmpl")
         if template_file.exists():
             return template_file
         print(f"Unable to find template: {template}\n")
@@ -121,11 +147,11 @@ class Command(ScrapyCommand):
     def _list_templates(self):
         print("Available templates:")
         for file in sorted(Path(self.templates_dir).iterdir()):
-            if file.suffix == '.tmpl':
+            if file.suffix == ".tmpl":
                 print(f"  {file.stem}")
 
     def _spider_exists(self, name: str) -> bool:
-        if not self.settings.get('NEWSPIDER_MODULE'):
+        if not self.settings.get("NEWSPIDER_MODULE"):
             # if run as a standalone command and file with same filename already exists
             path = Path(name + ".py")
             if path.exists():
@@ -148,7 +174,7 @@ class Command(ScrapyCommand):
             return True
 
         # a file with the same name exists in the target directory
-        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+        spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
         spiders_dir = Path(cast(str, spiders_module.__file__)).parent
         spiders_dir_abs = spiders_dir.resolve()
         path = spiders_dir_abs / (name + ".py")
@@ -160,7 +186,9 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self) -> str:
-        return str(Path(
-            self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
-            'spiders'
-        ))
+        return str(
+            Path(
+                self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
+                "spiders",
+            )
+        )
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 54d7bb228..2a81bd182 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -4,7 +4,7 @@ from scrapy.commands import ScrapyCommand
 class Command(ScrapyCommand):
 
     requires_project = True
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {"LOG_ENABLED": False}
 
     def short_desc(self):
         return "List available spiders"
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index c2df7f8a5..ac97b6193 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -32,28 +32,72 @@ class Command(BaseRunSpiderCommand):
 
     def add_options(self, parser):
         BaseRunSpiderCommand.add_options(self, parser)
-        parser.add_argument("--spider", dest="spider", default=None,
-                            help="use this spider without looking for one")
-        parser.add_argument("--pipelines", action="store_true",
-                            help="process items through pipelines")
-        parser.add_argument("--nolinks", dest="nolinks", action="store_true",
-                            help="don't show links to follow (extracted requests)")
-        parser.add_argument("--noitems", dest="noitems", action="store_true",
-                            help="don't show scraped items")
-        parser.add_argument("--nocolour", dest="nocolour", action="store_true",
-                            help="avoid using pygments to colorize the output")
-        parser.add_argument("-r", "--rules", dest="rules", action="store_true",
-                            help="use CrawlSpider rules to discover the callback")
-        parser.add_argument("-c", "--callback", dest="callback",
-                            help="use this callback for parsing, instead looking for a callback")
-        parser.add_argument("-m", "--meta", dest="meta",
-                            help="inject extra meta into the Request, it must be a valid raw json string")
-        parser.add_argument("--cbkwargs", dest="cbkwargs",
-                            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
-        parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help="maximum depth for parsing requests [default: %(default)s]")
-        parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
-                            help="print each depth level one by one")
+        parser.add_argument(
+            "--spider",
+            dest="spider",
+            default=None,
+            help="use this spider without looking for one",
+        )
+        parser.add_argument(
+            "--pipelines", action="store_true", help="process items through pipelines"
+        )
+        parser.add_argument(
+            "--nolinks",
+            dest="nolinks",
+            action="store_true",
+            help="don't show links to follow (extracted requests)",
+        )
+        parser.add_argument(
+            "--noitems",
+            dest="noitems",
+            action="store_true",
+            help="don't show scraped items",
+        )
+        parser.add_argument(
+            "--nocolour",
+            dest="nocolour",
+            action="store_true",
+            help="avoid using pygments to colorize the output",
+        )
+        parser.add_argument(
+            "-r",
+            "--rules",
+            dest="rules",
+            action="store_true",
+            help="use CrawlSpider rules to discover the callback",
+        )
+        parser.add_argument(
+            "-c",
+            "--callback",
+            dest="callback",
+            help="use this callback for parsing, instead looking for a callback",
+        )
+        parser.add_argument(
+            "-m",
+            "--meta",
+            dest="meta",
+            help="inject extra meta into the Request, it must be a valid raw json string",
+        )
+        parser.add_argument(
+            "--cbkwargs",
+            dest="cbkwargs",
+            help="inject extra callback kwargs into the Request, it must be a valid raw json string",
+        )
+        parser.add_argument(
+            "-d",
+            "--depth",
+            dest="depth",
+            type=int,
+            default=1,
+            help="maximum depth for parsing requests [default: %(default)s]",
+        )
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            dest="verbose",
+            action="store_true",
+            help="print each depth level one by one",
+        )
 
     @property
     def max_level(self):
@@ -98,13 +142,13 @@ class Command(BaseRunSpiderCommand):
 
         if opts.verbose:
             for level in range(1, self.max_level + 1):
-                print(f'\n>>> DEPTH LEVEL: {level} <<<')
+                print(f"\n>>> DEPTH LEVEL: {level} <<<")
                 if not opts.noitems:
                     self.print_items(level, colour)
                 if not opts.nolinks:
                     self.print_requests(level, colour)
         else:
-            print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
+            print(f"\n>>> STATUS DEPTH LEVEL {self.max_level} <<<")
             if not opts.noitems:
                 self.print_items(colour=colour)
             if not opts.nolinks:
@@ -125,14 +169,16 @@ class Command(BaseRunSpiderCommand):
         return d
 
     def get_callback_from_rules(self, spider, response):
-        if getattr(spider, 'rules', None):
+        if getattr(spider, "rules", None):
             for rule in spider.rules:
                 if rule.link_extractor.matches(response.url):
                     return rule.callback or "parse"
         else:
-            logger.error('No CrawlSpider rules found in spider %(spider)r, '
-                         'please specify a callback to use for parsing',
-                         {'spider': spider.name})
+            logger.error(
+                "No CrawlSpider rules found in spider %(spider)r, "
+                "please specify a callback to use for parsing",
+                {"spider": spider.name},
+            )
 
     def set_spidercls(self, url, opts):
         spider_loader = self.crawler_process.spider_loader
@@ -140,15 +186,17 @@ class Command(BaseRunSpiderCommand):
             try:
                 self.spidercls = spider_loader.load(opts.spider)
             except KeyError:
-                logger.error('Unable to find spider: %(spider)s',
-                             {'spider': opts.spider})
+                logger.error(
+                    "Unable to find spider: %(spider)s", {"spider": opts.spider}
+                )
         else:
             self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
-                logger.error('Unable to find spider for: %(url)s', {'url': url})
+                logger.error("Unable to find spider for: %(url)s", {"url": url})
 
         def _start_requests(spider):
             yield self.prepare_request(spider, Request(url), opts)
+
         if self.spidercls:
             self.spidercls.start_requests = _start_requests
 
@@ -158,8 +206,7 @@ class Command(BaseRunSpiderCommand):
         self.crawler_process.start()
 
         if not self.first_response:
-            logger.error('No response downloaded for: %(url)s',
-                         {'url': url})
+            logger.error("No response downloaded for: %(url)s", {"url": url})
 
     def scraped_data(self, args):
         items, requests, opts, depth, spider, callback = args
@@ -173,8 +220,8 @@ class Command(BaseRunSpiderCommand):
         scraped_data = items if opts.output else []
         if depth < opts.depth:
             for req in requests:
-                req.meta['_depth'] = depth + 1
-                req.meta['_callback'] = req.callback
+                req.meta["_depth"] = depth + 1
+                req.meta["_callback"] = req.callback
                 req.callback = callback
             scraped_data += requests
 
@@ -187,7 +234,7 @@ class Command(BaseRunSpiderCommand):
                 self.first_response = response
 
             # determine real callback
-            cb = response.meta['_callback']
+            cb = response.meta["_callback"]
             if not cb:
                 if opts.callback:
                     cb = opts.callback
@@ -195,23 +242,27 @@ class Command(BaseRunSpiderCommand):
                     cb = self.get_callback_from_rules(spider, response)
 
                     if not cb:
-                        logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
-                                     {'url': response.url, 'spider': spider.name})
+                        logger.error(
+                            "Cannot find a rule that matches %(url)r in spider: %(spider)s",
+                            {"url": response.url, "spider": spider.name},
+                        )
                         return
                 else:
-                    cb = 'parse'
+                    cb = "parse"
 
             if not callable(cb):
                 cb_method = getattr(spider, cb, None)
                 if callable(cb_method):
                     cb = cb_method
                 else:
-                    logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
-                                 {'callback': cb, 'spider': spider.name})
+                    logger.error(
+                        "Cannot find callback %(callback)r in spider: %(spider)s",
+                        {"callback": cb, "spider": spider.name},
+                    )
                     return
 
             # parse items and requests
-            depth = response.meta['_depth']
+            depth = response.meta["_depth"]
 
             d = self.run_callback(response, cb, cb_kwargs)
             d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
@@ -226,8 +277,8 @@ class Command(BaseRunSpiderCommand):
         if opts.cbkwargs:
             request.cb_kwargs.update(opts.cbkwargs)
 
-        request.meta['_depth'] = 1
-        request.meta['_callback'] = request.callback
+        request.meta["_depth"] = 1
+        request.meta["_callback"] = request.callback
         request.callback = callback
         return request
 
@@ -242,16 +293,22 @@ class Command(BaseRunSpiderCommand):
             try:
                 opts.meta = json.loads(opts.meta)
             except ValueError:
-                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
-                                 "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError(
+                    "Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+                    'Example: --meta=\'{"foo" : "bar"}\'',
+                    print_help=False,
+                )
 
     def process_request_cb_kwargs(self, opts):
         if opts.cbkwargs:
             try:
                 opts.cbkwargs = json.loads(opts.cbkwargs)
             except ValueError:
-                raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
-                                 "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError(
+                    "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+                    'Example: --cbkwargs=\'{"foo" : "bar"}\'',
+                    print_help=False,
+                )
 
     def run(self, args, opts):
         # parse arguments
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 22fa6a53d..9751c6c30 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -12,7 +12,7 @@ from scrapy.commands import BaseRunSpiderCommand
 
 def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
     abspath = Path(filepath).resolve()
-    if abspath.suffix not in ('.py', '.pyw'):
+    if abspath.suffix not in (".py", ".pyw"):
         raise ValueError(f"Not a Python source file: {abspath}")
     dirname = str(abspath.parent)
     sys.path = [dirname] + sys.path
@@ -26,7 +26,7 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
 class Command(BaseRunSpiderCommand):
 
     requires_project = False
-    default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
+    default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
 
     def syntax(self):
         return "[options] <spider_file>"
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 1b2e2601e..c43298ac7 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -7,8 +7,7 @@ from scrapy.settings import BaseSettings
 class Command(ScrapyCommand):
 
     requires_project = False
-    default_settings = {'LOG_ENABLED': False,
-                        'SPIDER_LOADER_WARN_ONLY': True}
+    default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
     def syntax(self):
         return "[options]"
@@ -18,16 +17,33 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_argument("--get", dest="get", metavar="SETTING",
-                            help="print raw setting value")
-        parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
-                            help="print setting value, interpreted as a boolean")
-        parser.add_argument("--getint", dest="getint", metavar="SETTING",
-                            help="print setting value, interpreted as an integer")
-        parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
-                            help="print setting value, interpreted as a float")
-        parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
-                            help="print setting value, interpreted as a list")
+        parser.add_argument(
+            "--get", dest="get", metavar="SETTING", help="print raw setting value"
+        )
+        parser.add_argument(
+            "--getbool",
+            dest="getbool",
+            metavar="SETTING",
+            help="print setting value, interpreted as a boolean",
+        )
+        parser.add_argument(
+            "--getint",
+            dest="getint",
+            metavar="SETTING",
+            help="print setting value, interpreted as an integer",
+        )
+        parser.add_argument(
+            "--getfloat",
+            dest="getfloat",
+            metavar="SETTING",
+            help="print setting value, interpreted as a float",
+        )
+        parser.add_argument(
+            "--getlist",
+            dest="getlist",
+            metavar="SETTING",
+            help="print setting value, interpreted as a list",
+        )
 
     def run(self, args, opts):
         settings = self.crawler_process.settings
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index f67a5886a..1fad8f328 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -16,9 +16,9 @@ class Command(ScrapyCommand):
 
     requires_project = False
     default_settings = {
-        'KEEP_ALIVE': True,
-        'LOGSTATS_INTERVAL': 0,
-        'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
+        "KEEP_ALIVE": True,
+        "LOGSTATS_INTERVAL": 0,
+        "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
     }
 
     def syntax(self):
@@ -28,17 +28,26 @@ class Command(ScrapyCommand):
         return "Interactive scraping console"
 
     def long_desc(self):
-        return ("Interactive console for scraping the given url or file. "
-                "Use ./file.html syntax or full path for local file.")
+        return (
+            "Interactive console for scraping the given url or file. "
+            "Use ./file.html syntax or full path for local file."
+        )
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_argument("-c", dest="code",
-                            help="evaluate the code in the shell, print the result and exit")
-        parser.add_argument("--spider", dest="spider",
-                            help="use this spider")
-        parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
-                            help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_argument(
+            "-c",
+            dest="code",
+            help="evaluate the code in the shell, print the result and exit",
+        )
+        parser.add_argument("--spider", dest="spider", help="use this spider")
+        parser.add_argument(
+            "--no-redirect",
+            dest="no_redirect",
+            action="store_true",
+            default=False,
+            help="do not handle HTTP 3xx status codes and print response as-is",
+        )
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
@@ -58,8 +67,9 @@ class Command(ScrapyCommand):
         if opts.spider:
             spidercls = spider_loader.load(opts.spider)
         elif url:
-            spidercls = spidercls_for_request(spider_loader, Request(url),
-                                              spidercls, log_multiple=True)
+            spidercls = spidercls_for_request(
+                spider_loader, Request(url), spidercls, log_multiple=True
+            )
 
         # The crawler is created this way since the Shell manually handles the
         # crawling engine, so the set up in the crawl method won't work
@@ -74,7 +84,9 @@ class Command(ScrapyCommand):
         shell.start(url=url, redirect=not opts.no_redirect)
 
     def _start_crawler_thread(self):
-        t = Thread(target=self.crawler_process.start,
-                   kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
+        t = Thread(
+            target=self.crawler_process.start,
+            kwargs={"stop_after_crawl": False, "install_signal_handlers": False},
+        )
         t.daemon = True
         t.start()
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 9e1e95ab1..3ed1f5dbc 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -13,14 +13,14 @@ from scrapy.exceptions import UsageError
 
 
 TEMPLATES_TO_RENDER = (
-    ('scrapy.cfg',),
-    ('${project_name}', 'settings.py.tmpl'),
-    ('${project_name}', 'items.py.tmpl'),
-    ('${project_name}', 'pipelines.py.tmpl'),
-    ('${project_name}', 'middlewares.py.tmpl'),
+    ("scrapy.cfg",),
+    ("${project_name}", "settings.py.tmpl"),
+    ("${project_name}", "items.py.tmpl"),
+    ("${project_name}", "pipelines.py.tmpl"),
+    ("${project_name}", "middlewares.py.tmpl"),
 )
 
-IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
 
 
 def _make_writable(path):
@@ -31,8 +31,7 @@ def _make_writable(path):
 class Command(ScrapyCommand):
 
     requires_project = False
-    default_settings = {'LOG_ENABLED': False,
-                        'SPIDER_LOADER_WARN_ONLY': True}
+    default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
     def syntax(self):
         return "<project_name> [project_dir]"
@@ -45,11 +44,13 @@ class Command(ScrapyCommand):
             spec = find_spec(module_name)
             return spec is not None and spec.loader is not None
 
-        if not re.search(r'^[_a-zA-Z]\w*$', project_name):
-            print('Error: Project names must begin with a letter and contain'
-                  ' only\nletters, numbers and underscores')
+        if not re.search(r"^[_a-zA-Z]\w*$", project_name):
+            print(
+                "Error: Project names must begin with a letter and contain"
+                " only\nletters, numbers and underscores"
+            )
         elif _module_exists(project_name):
-            print(f'Error: Module {project_name!r} already exists')
+            print(f"Error: Module {project_name!r} already exists")
         else:
             return True
         return False
@@ -96,9 +97,9 @@ class Command(ScrapyCommand):
         else:
             project_dir = Path(args[0])
 
-        if (project_dir / 'scrapy.cfg').exists():
+        if (project_dir / "scrapy.cfg").exists():
             self.exitcode = 1
-            print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')
+            print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}")
             return
 
         if not self._is_valid_name(project_name):
@@ -106,12 +107,24 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(Path(self.templates_dir), project_dir.resolve())
-        move(project_dir / 'module', project_dir / project_name)
+        move(project_dir / "module", project_dir / project_name)
         for paths in TEMPLATES_TO_RENDER:
-            tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))
-            render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
-        print(f"New Scrapy project '{project_name}', using template directory "
-              f"'{self.templates_dir}', created in:")
+            tplfile = Path(
+                project_dir,
+                *(
+                    string.Template(s).substitute(project_name=project_name)
+                    for s in paths
+                ),
+            )
+            render_templatefile(
+                tplfile,
+                project_name=project_name,
+                ProjectName=string_camelcase(project_name),
+            )
+        print(
+            f"New Scrapy project '{project_name}', using template directory "
+            f"'{self.templates_dir}', created in:"
+        )
         print(f"    {project_dir.resolve()}\n")
         print("You can start your first spider with:")
         print(f"    cd {project_dir}")
@@ -119,7 +132,9 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self) -> str:
-        return str(Path(
-            self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
-            'project'
-        ))
+        return str(
+            Path(
+                self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
+                "project",
+            )
+        )
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index c6a3c273a..faa0e242c 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -5,8 +5,7 @@ from scrapy.utils.versions import scrapy_components_versions
 
 class Command(ScrapyCommand):
 
-    default_settings = {'LOG_ENABLED': False,
-                        'SPIDER_LOADER_WARN_ONLY': True}
+    default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
     def syntax(self):
         return "[-v]"
@@ -16,8 +15,13 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
-                            help="also display twisted/python/platform info (useful for bug reports)")
+        parser.add_argument(
+            "--verbose",
+            "-v",
+            dest="verbose",
+            action="store_true",
+            help="also display twisted/python/platform info (useful for bug reports)",
+        )
 
     def run(self, args, opts):
         if opts.verbose:
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index b1f52abe2..a81af7565 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -4,16 +4,17 @@ from scrapy.utils.response import open_in_browser
 
 
 class Command(fetch.Command):
-
     def short_desc(self):
         return "Open URL in browser, as seen by Scrapy"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
+        return (
+            "Fetch a URL using the Scrapy downloader and show its contents in a browser"
+        )
 
     def add_options(self, parser):
         super().add_options(parser)
-        parser.add_argument('--headers', help=argparse.SUPPRESS)
+        parser.add_argument("--headers", help=argparse.SUPPRESS)
 
     def _print_response(self, response, opts):
         open_in_browser(response)
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index b47e55092..86098edca 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -11,16 +11,17 @@ from scrapy.utils.spider import iterate_spider_output
 
 
 class Contract:
-    """ Abstract class for contracts """
+    """Abstract class for contracts"""
+
     request_cls = None
 
     def __init__(self, method, *args):
-        self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
-        self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
+        self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
+        self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
         self.args = args
 
     def add_pre_hook(self, request, results):
-        if hasattr(self, 'pre_process'):
+        if hasattr(self, "pre_process"):
             cb = request.callback
 
             @wraps(cb)
@@ -43,7 +44,7 @@ class Contract:
         return request
 
     def add_post_hook(self, request, results):
-        if hasattr(self, 'post_process'):
+        if hasattr(self, "post_process"):
             cb = request.callback
 
             @wraps(cb)
@@ -88,12 +89,12 @@ class ContractsManager:
 
     def extract_contracts(self, method):
         contracts = []
-        for line in method.__doc__.split('\n'):
+        for line in method.__doc__.split("\n"):
             line = line.strip()
 
-            if line.startswith('@'):
-                name, args = re.match(r'@(\w+)\s*(.*)', line).groups()
-                args = re.split(r'\s+', args)
+            if line.startswith("@"):
+                name, args = re.match(r"@(\w+)\s*(.*)", line).groups()
+                args = re.split(r"\s+", args)
 
                 contracts.append(self.contracts[name](method, *args))
 
@@ -106,7 +107,7 @@ class ContractsManager:
             try:
                 requests.append(self.from_method(bound_method, results))
             except Exception:
-                case = _create_testcase(bound_method, 'contract')
+                case = _create_testcase(bound_method, "contract")
                 results.addError(case, sys.exc_info())
 
         return requests
@@ -124,13 +125,13 @@ class ContractsManager:
 
             # Don't filter requests to allow
             # testing different callbacks on the same URL.
-            kwargs['dont_filter'] = True
-            kwargs['callback'] = method
+            kwargs["dont_filter"] = True
+            kwargs["callback"] = method
 
             for contract in contracts:
                 kwargs = contract.adjust_request_args(kwargs)
 
-            args.remove('self')
+            args.remove("self")
 
             # check if all positional arguments are defined in kwargs
             if set(args).issubset(set(kwargs)):
@@ -146,7 +147,7 @@ class ContractsManager:
                 return request
 
     def _clean_req(self, request, method, results):
-        """ stop the request from returning objects and records any errors """
+        """stop the request from returning objects and records any errors"""
 
         cb = request.callback
 
@@ -156,11 +157,11 @@ class ContractsManager:
                 output = cb(response, **cb_kwargs)
                 output = list(iterate_spider_output(output))
             except Exception:
-                case = _create_testcase(method, 'callback')
+                case = _create_testcase(method, "callback")
                 results.addError(case, sys.exc_info())
 
         def eb_wrapper(failure):
-            case = _create_testcase(method, 'errback')
+            case = _create_testcase(method, "errback")
             exc_info = failure.type, failure.value, failure.getTracebackObject()
             results.addError(case, exc_info)
 
@@ -175,6 +176,6 @@ def _create_testcase(method, desc):
         def __str__(_self):
             return f"[{spider}] {method.__name__} ({desc})"
 
-    name = f'{spider}_{method.__name__}'
+    name = f"{spider}_{method.__name__}"
     setattr(ContractTestCase, name, lambda x: x)
     return ContractTestCase(name)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 9704f5253..e41d83960 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -9,50 +9,50 @@ from scrapy.http import Request
 
 # contracts
 class UrlContract(Contract):
-    """ Contract to set the url of the request (mandatory)
-        @url http://scrapy.org
+    """Contract to set the url of the request (mandatory)
+    @url http://scrapy.org
     """
 
-    name = 'url'
+    name = "url"
 
     def adjust_request_args(self, args):
-        args['url'] = self.args[0]
+        args["url"] = self.args[0]
         return args
 
 
 class CallbackKeywordArgumentsContract(Contract):
-    """ Contract to set the keyword arguments for the request.
-        The value should be a JSON-encoded dictionary, e.g.:
+    """Contract to set the keyword arguments for the request.
+    The value should be a JSON-encoded dictionary, e.g.:
 
-        @cb_kwargs {"arg1": "some value"}
+    @cb_kwargs {"arg1": "some value"}
     """
 
-    name = 'cb_kwargs'
+    name = "cb_kwargs"
 
     def adjust_request_args(self, args):
-        args['cb_kwargs'] = json.loads(' '.join(self.args))
+        args["cb_kwargs"] = json.loads(" ".join(self.args))
         return args
 
 
 class ReturnsContract(Contract):
-    """ Contract to check the output of a callback
+    """Contract to check the output of a callback
 
-        general form:
-        @returns request(s)/item(s) [min=1 [max]]
+    general form:
+    @returns request(s)/item(s) [min=1 [max]]
 
-        e.g.:
-        @returns request
-        @returns request 2
-        @returns request 2 10
-        @returns request 0 10
+    e.g.:
+    @returns request
+    @returns request 2
+    @returns request 2 10
+    @returns request 0 10
     """
 
-    name = 'returns'
+    name = "returns"
     object_type_verifiers = {
-        'request': lambda x: isinstance(x, Request),
-        'requests': lambda x: isinstance(x, Request),
-        'item': is_item,
-        'items': is_item,
+        "request": lambda x: isinstance(x, Request),
+        "requests": lambda x: isinstance(x, Request),
+        "item": is_item,
+        "items": is_item,
     }
 
     def __init__(self, *args, **kwargs):
@@ -73,7 +73,7 @@ class ReturnsContract(Contract):
         try:
             self.max_bound = int(self.args[2])
         except IndexError:
-            self.max_bound = float('inf')
+            self.max_bound = float("inf")
 
     def post_process(self, output):
         occurrences = 0
@@ -81,23 +81,25 @@ class ReturnsContract(Contract):
             if self.obj_type_verifier(x):
                 occurrences += 1
 
-        assertion = (self.min_bound <= occurrences <= self.max_bound)
+        assertion = self.min_bound <= occurrences <= self.max_bound
 
         if not assertion:
             if self.min_bound == self.max_bound:
                 expected = self.min_bound
             else:
-                expected = f'{self.min_bound}..{self.max_bound}'
+                expected = f"{self.min_bound}..{self.max_bound}"
 
-            raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
+            raise ContractFail(
+                f"Returned {occurrences} {self.obj_name}, expected {expected}"
+            )
 
 
 class ScrapesContract(Contract):
-    """ Contract to check presence of fields in scraped items
-        @scrapes page_name page_body
+    """Contract to check presence of fields in scraped items
+    @scrapes page_name page_body
     """
 
-    name = 'scrapes'
+    name = "scrapes"
 
     def post_process(self, output):
         for x in output:
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 4f7ab594f..3a7de8072 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -41,9 +41,11 @@ class Slot:
 
     def __repr__(self):
         cls_name = self.__class__.__name__
-        return (f"{cls_name}(concurrency={self.concurrency!r}, "
-                f"delay={self.delay:.2f}, "
-                f"randomize_delay={self.randomize_delay!r})")
+        return (
+            f"{cls_name}(concurrency={self.concurrency!r}, "
+            f"delay={self.delay:.2f}, "
+            f"randomize_delay={self.randomize_delay!r})"
+        )
 
     def __str__(self):
         return (
@@ -56,11 +58,11 @@ class Slot:
 
 
 def _get_concurrency_delay(concurrency, spider, settings):
-    delay = settings.getfloat('DOWNLOAD_DELAY')
-    if hasattr(spider, 'download_delay'):
+    delay = settings.getfloat("DOWNLOAD_DELAY")
+    if hasattr(spider, "download_delay"):
         delay = spider.download_delay
 
-    if hasattr(spider, 'max_concurrent_requests'):
+    if hasattr(spider, "max_concurrent_requests"):
         concurrency = spider.max_concurrent_requests
 
     return concurrency, delay
@@ -68,7 +70,7 @@ def _get_concurrency_delay(concurrency, spider, settings):
 
 class Downloader:
 
-    DOWNLOAD_SLOT = 'download_slot'
+    DOWNLOAD_SLOT = "download_slot"
 
     def __init__(self, crawler):
         self.settings = crawler.settings
@@ -76,10 +78,10 @@ class Downloader:
         self.slots = {}
         self.active = set()
         self.handlers = DownloadHandlers(crawler)
-        self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
-        self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
-        self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
-        self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
+        self.total_concurrency = self.settings.getint("CONCURRENT_REQUESTS")
+        self.domain_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
+        self.ip_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
+        self.randomize_delay = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
         self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
         self._slot_gc_loop = task.LoopingCall(self._slot_gc)
         self._slot_gc_loop.start(60)
@@ -99,7 +101,9 @@ class Downloader:
     def _get_slot(self, request, spider):
         key = self._get_slot_key(request, spider)
         if key not in self.slots:
-            conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
+            conc = (
+                self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
+            )
             conc, delay = _get_concurrency_delay(conc, spider, self.settings)
             self.slots[key] = Slot(conc, delay, self.randomize_delay)
 
@@ -109,7 +113,7 @@ class Downloader:
         if self.DOWNLOAD_SLOT in request.meta:
             return request.meta[self.DOWNLOAD_SLOT]
 
-        key = urlparse_cached(request).hostname or ''
+        key = urlparse_cached(request).hostname or ""
         if self.ip_concurrency:
             key = dnscache.get(key, key)
 
@@ -124,9 +128,9 @@ class Downloader:
             return response
 
         slot.active.add(request)
-        self.signals.send_catch_log(signal=signals.request_reached_downloader,
-                                    request=request,
-                                    spider=spider)
+        self.signals.send_catch_log(
+            signal=signals.request_reached_downloader, request=request, spider=spider
+        )
         deferred = defer.Deferred().addBoth(_deactivate)
         slot.queue.append((request, deferred))
         self._process_queue(spider, slot)
@@ -134,6 +138,7 @@ class Downloader:
 
     def _process_queue(self, spider, slot):
         from twisted.internet import reactor
+
         if slot.latercall and slot.latercall.active():
             return
 
@@ -143,7 +148,9 @@ class Downloader:
         if delay:
             penalty = delay - now + slot.lastseen
             if penalty > 0:
-                slot.latercall = reactor.callLater(penalty, self._process_queue, spider, slot)
+                slot.latercall = reactor.callLater(
+                    penalty, self._process_queue, spider, slot
+                )
                 return
 
         # Process enqueued requests if there are free slots to transfer for this slot
@@ -166,11 +173,14 @@ class Downloader:
         # 2. Notify response_downloaded listeners about the recent download
         # before querying queue for next request
         def _downloaded(response):
-            self.signals.send_catch_log(signal=signals.response_downloaded,
-                                        response=response,
-                                        request=request,
-                                        spider=spider)
+            self.signals.send_catch_log(
+                signal=signals.response_downloaded,
+                response=response,
+                request=request,
+                spider=spider,
+            )
             return response
+
         dfd.addCallback(_downloaded)
 
         # 3. After response arrives, remove the request from transferring
@@ -182,9 +192,9 @@ class Downloader:
         def finish_transferring(_):
             slot.transferring.remove(request)
             self._process_queue(spider, slot)
-            self.signals.send_catch_log(signal=signals.request_left_downloader,
-                                        request=request,
-                                        spider=spider)
+            self.signals.send_catch_log(
+                signal=signals.request_left_downloader, request=request, spider=spider
+            )
             return _
 
         return dfd.addBoth(finish_transferring)
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 4abde2238..9dd2df638 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -2,13 +2,22 @@ import warnings
 
 from OpenSSL import SSL
 from twisted.internet._sslverify import _setAcceptableProtocols
-from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
+from twisted.internet.ssl import (
+    optionsForClientTLS,
+    CertificateOptions,
+    platformTrust,
+    AcceptableCiphers,
+)
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
 from zope.interface.verify import verifyObject
 
-from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
+from scrapy.core.downloader.tls import (
+    DEFAULT_CIPHERS,
+    openssl_methods,
+    ScrapyClientTLSOptions,
+)
 from scrapy.utils.misc import create_instance, load_object
 
 
@@ -24,7 +33,14 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
      understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
     """
 
-    def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
+    def __init__(
+        self,
+        method=SSL.SSLv23_METHOD,
+        tls_verbose_logging=False,
+        tls_ciphers=None,
+        *args,
+        **kwargs,
+    ):
         super().__init__(*args, **kwargs)
         self._ssl_method = method
         self.tls_verbose_logging = tls_verbose_logging
@@ -35,9 +51,15 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
 
     @classmethod
     def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
-        tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
-        tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
-        return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
+        tls_verbose_logging = settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING")
+        tls_ciphers = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
+        return cls(
+            method=method,
+            tls_verbose_logging=tls_verbose_logging,
+            tls_ciphers=tls_ciphers,
+            *args,
+            **kwargs,
+        )
 
     def getCertificateOptions(self):
         # setting verify=True will require you to provide CAs
@@ -53,7 +75,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         #   not calling super().__init__
         return CertificateOptions(
             verify=False,
-            method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
+            method=getattr(self, "method", getattr(self, "_ssl_method", None)),
             fixBrokenPeers=True,
             acceptableCiphers=self.tls_ciphers,
         )
@@ -64,8 +86,11 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         return self.getCertificateOptions().getContext()
 
     def creatorForNetloc(self, hostname, port):
-        return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
-                                      verbose_logging=self.tls_verbose_logging)
+        return ScrapyClientTLSOptions(
+            hostname.decode("ascii"),
+            self.getContext(),
+            verbose_logging=self.tls_verbose_logging,
+        )
 
 
 @implementer(IPolicyForHTTPS)
@@ -95,7 +120,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
         return optionsForClientTLS(
             hostname=hostname.decode("ascii"),
             trustRoot=platformTrust(),
-            extraCertificateOptions={'method': self._ssl_method},
+            extraCertificateOptions={"method": self._ssl_method},
         )
 
 
@@ -118,8 +143,8 @@ class AcceptableProtocolsContextFactory:
 
 
 def load_context_factory_from_settings(settings, crawler):
-    ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
-    context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+    ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
+    context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
     # try method-aware context factory
     try:
         context_factory = create_instance(
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 73aeb2352..bb2141d28 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -15,14 +15,14 @@ logger = logging.getLogger(__name__)
 
 
 class DownloadHandlers:
-
     def __init__(self, crawler):
         self._crawler = crawler
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
         handlers = without_none_values(
-            crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
+            crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
+        )
         for scheme, clspath in handlers.items():
             self._schemes[scheme] = clspath
             self._load_handler(scheme, skip_lazy=True)
@@ -38,7 +38,7 @@ class DownloadHandlers:
         if scheme in self._notconfigured:
             return None
         if scheme not in self._schemes:
-            self._notconfigured[scheme] = 'no handler available for that scheme'
+            self._notconfigured[scheme] = "no handler available for that scheme"
             return None
 
         return self._load_handler(scheme)
@@ -47,7 +47,7 @@ class DownloadHandlers:
         path = self._schemes[scheme]
         try:
             dhcls = load_object(path)
-            if skip_lazy and getattr(dhcls, 'lazy', True):
+            if skip_lazy and getattr(dhcls, "lazy", True):
                 return None
             dh = create_instance(
                 objcls=dhcls,
@@ -58,9 +58,12 @@ class DownloadHandlers:
             self._notconfigured[scheme] = str(ex)
             return None
         except Exception as ex:
-            logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
-                         {"clspath": path, "scheme": scheme},
-                         exc_info=True, extra={'crawler': self._crawler})
+            logger.error(
+                'Loading "%(clspath)s" for scheme "%(scheme)s"',
+                {"clspath": path, "scheme": scheme},
+                exc_info=True,
+                extra={"crawler": self._crawler},
+            )
             self._notconfigured[scheme] = str(ex)
             return None
         else:
@@ -71,11 +74,13 @@ class DownloadHandlers:
         scheme = urlparse_cached(request).scheme
         handler = self._get_handler(scheme)
         if not handler:
-            raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
+            raise NotSupported(
+                f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
+            )
         return handler.download_request(request, spider)
 
     @defer.inlineCallbacks
     def _close(self, *_a, **_kw):
         for dh in self._handlers.values():
-            if hasattr(dh, 'close'):
+            if hasattr(dh, "close"):
                 yield dh.close()
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index a45b4ff3c..8b78c53c1 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -14,9 +14,8 @@ class DataURIDownloadHandler:
         respcls = responsetypes.from_mimetype(uri.media_type)
 
         resp_kwargs = {}
-        if (issubclass(respcls, TextResponse)
-                and uri.media_type.split('/')[0] == 'text'):
-            charset = uri.media_type_parameters.get('charset')
-            resp_kwargs['encoding'] = charset
+        if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
+            charset = uri.media_type_parameters.get("charset")
+            resp_kwargs["encoding"] = charset
 
         return respcls(url=request.url, body=uri.data, **resp_kwargs)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index a495874bd..4081545ce 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -71,9 +71,9 @@ class FTPDownloadHandler:
     }
 
     def __init__(self, settings):
-        self.default_user = settings['FTP_USER']
-        self.default_password = settings['FTP_PASSWORD']
-        self.passive_mode = settings['FTP_PASSIVE_MODE']
+        self.default_user = settings["FTP_USER"]
+        self.default_password = settings["FTP_PASSWORD"]
+        self.passive_mode = settings["FTP_PASSIVE_MODE"]
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -81,12 +81,16 @@ class FTPDownloadHandler:
 
     def download_request(self, request, spider):
         from twisted.internet import reactor
+
         parsed_url = urlparse_cached(request)
         user = request.meta.get("ftp_user", self.default_user)
         password = request.meta.get("ftp_password", self.default_password)
-        passive_mode = 1 if bool(request.meta.get("ftp_passive",
-                                                  self.passive_mode)) else 0
-        creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
+        passive_mode = (
+            1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0
+        )
+        creator = ClientCreator(
+            reactor, FTPClient, user, password, passive=passive_mode
+        )
         dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
         return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
 
@@ -103,7 +107,7 @@ class FTPDownloadHandler:
     def _build_response(self, result, request, protocol):
         self.result = result
         protocol.close()
-        headers = {"local filename": protocol.filename or '', "size": protocol.size}
+        headers = {"local filename": protocol.filename or "", "size": protocol.size}
         body = to_bytes(protocol.filename or protocol.body.read())
         respcls = responsetypes.from_args(url=request.url, body=body)
         return respcls(url=request.url, status=200, body=body, headers=headers)
@@ -115,5 +119,7 @@ class FTPDownloadHandler:
             if m:
                 ftpcode = m.group()
                 httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
-                return Response(url=request.url, status=httpcode, body=to_bytes(message))
+                return Response(
+                    url=request.url, status=httpcode, body=to_bytes(message)
+                )
         raise result.type(result.value)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index a75532d2a..6c1dac4a5 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -8,8 +8,10 @@ class HTTP10DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
-        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
+        self.ClientContextFactory = load_object(
+            settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
+        )
         self._settings = settings
         self._crawler = crawler
 
@@ -25,8 +27,9 @@ class HTTP10DownloadHandler:
 
     def _connect(self, factory):
         from twisted.internet import reactor
+
         host, port = to_unicode(factory.host), factory.port
-        if factory.scheme == b'https':
+        if factory.scheme == b"https":
             client_context_factory = create_instance(
                 objcls=self.ClientContextFactory,
                 settings=self._settings,
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 1c98e60e1..201c84ff8 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -12,7 +12,13 @@ from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.python.failure import Failure
-from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
+from twisted.web.client import (
+    Agent,
+    HTTPConnectionPool,
+    ResponseDone,
+    ResponseFailed,
+    URI,
+)
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
@@ -36,14 +42,17 @@ class HTTP11DownloadHandler:
         self._crawler = crawler
 
         from twisted.internet import reactor
+
         self._pool = HTTPConnectionPool(reactor, persistent=True)
-        self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
+        self._pool.maxPersistentPerHost = settings.getint(
+            "CONCURRENT_REQUESTS_PER_DOMAIN"
+        )
         self._pool._factory.noisy = False
 
         self._contextFactory = load_context_factory_from_settings(settings, crawler)
-        self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
-        self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
-        self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
+        self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE")
+        self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE")
+        self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
         self._disconnect_timeout = 1
 
     @classmethod
@@ -55,8 +64,8 @@ class HTTP11DownloadHandler:
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,
             pool=self._pool,
-            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
-            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
+            maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
+            warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
             crawler=self._crawler,
         )
@@ -64,6 +73,7 @@ class HTTP11DownloadHandler:
 
     def close(self):
         from twisted.internet import reactor
+
         d = self._pool.closeCachedConnections()
         # closeCachedConnections will hang on network or server issues, so
         # we'll manually timeout the deferred.
@@ -96,11 +106,23 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     with this endpoint comes from the pool and a CONNECT has already been issued
     for it.
     """
+
     _truncatedLength = 1000
-    _responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
+    _responseAnswer = (
+        r"HTTP/1\.. (?P<status>\d{3})(?P<reason>.{," + str(_truncatedLength) + r"})"
+    )
     _responseMatcher = re.compile(_responseAnswer.encode())
 
-    def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
+    def __init__(
+        self,
+        reactor,
+        host,
+        port,
+        proxyConf,
+        contextFactory,
+        timeout=30,
+        bindAddress=None,
+    ):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
         super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
         self._tunnelReadyDeferred = defer.Deferred()
@@ -111,7 +133,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
-        tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)
+        tunnelReq = tunnel_request_data(
+            self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
+        )
         protocol.transport.write(tunnelReq)
         self._protocolDataReceived = protocol.dataReceived
         protocol.dataReceived = self.processProxyResponse
@@ -129,24 +153,30 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         # from the proxy so that we don't send those bytes to the TLS layer
         #
         # see https://github.com/scrapy/scrapy/issues/2491
-        if b'\r\n\r\n' not in self._connectBuffer:
+        if b"\r\n\r\n" not in self._connectBuffer:
             return
         self._protocol.dataReceived = self._protocolDataReceived
         respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
-        if respm and int(respm.group('status')) == 200:
+        if respm and int(respm.group("status")) == 200:
             # set proper Server Name Indication extension
-            sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
+            sslOptions = self._contextFactory.creatorForNetloc(
+                self._tunneledHost, self._tunneledPort
+            )
             self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
             if respm:
-                extra = {'status': int(respm.group('status')),
-                         'reason': respm.group('reason').strip()}
+                extra = {
+                    "status": int(respm.group("status")),
+                    "reason": respm.group("reason").strip(),
+                }
             else:
-                extra = rcvd_bytes[:self._truncatedLength]
+                extra = rcvd_bytes[: self._truncatedLength]
             self._tunnelReadyDeferred.errback(
-                TunnelError('Could not open CONNECT tunnel with proxy '
-                            f'{self._host}:{self._port} [{extra!r}]')
+                TunnelError(
+                    "Could not open CONNECT tunnel with proxy "
+                    f"{self._host}:{self._port} [{extra!r}]"
+                )
             )
 
     def connectFailed(self, reason):
@@ -173,12 +203,12 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
     >>> s(tunnel_request_data(b"example.com", "8090"))
     'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
     """
-    host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
-    tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
-    tunnel_req += b'Host: ' + host_value + b'\r\n'
+    host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
+    tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
+    tunnel_req += b"Host: " + host_value + b"\r\n"
     if proxy_auth_header:
-        tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
-    tunnel_req += b'\r\n'
+        tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n"
+    tunnel_req += b"\r\n"
     return tunnel_req
 
 
@@ -190,8 +220,15 @@ class TunnelingAgent(Agent):
     proxy involved.
     """
 
-    def __init__(self, reactor, proxyConf, contextFactory=None,
-                 connectTimeout=None, bindAddress=None, pool=None):
+    def __init__(
+        self,
+        reactor,
+        proxyConf,
+        contextFactory=None,
+        connectTimeout=None,
+        bindAddress=None,
+        pool=None,
+    ):
         super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
@@ -207,7 +244,9 @@ class TunnelingAgent(Agent):
             bindAddress=self._endpointFactory._bindAddress,
         )
 
-    def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
+    def _requestWithEndpoint(
+        self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
+    ):
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
@@ -224,8 +263,9 @@ class TunnelingAgent(Agent):
 
 
 class ScrapyProxyAgent(Agent):
-
-    def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
+    def __init__(
+        self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None
+    ):
         super().__init__(
             reactor=reactor,
             connectTimeout=connectTimeout,
@@ -257,8 +297,17 @@ class ScrapyAgent:
     _ProxyAgent = ScrapyProxyAgent
     _TunnelingAgent = TunnelingAgent
 
-    def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
+    def __init__(
+        self,
+        contextFactory=None,
+        connectTimeout=10,
+        bindAddress=None,
+        pool=None,
+        maxsize=0,
+        warnsize=0,
+        fail_on_dataloss=True,
+        crawler=None,
+    ):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -271,14 +320,15 @@ class ScrapyAgent:
 
     def _get_agent(self, request, timeout):
         from twisted.internet import reactor
-        bindaddress = request.meta.get('bindaddress') or self._bindAddress
-        proxy = request.meta.get('proxy')
+
+        bindaddress = request.meta.get("bindaddress") or self._bindAddress
+        proxy = request.meta.get("proxy")
         if proxy:
             proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
             proxyHost = to_unicode(proxyHost)
-            if scheme == b'https':
-                proxyAuth = request.headers.get(b'Proxy-Authorization', None)
+            if scheme == b"https":
+                proxyAuth = request.headers.get(b"Proxy-Authorization", None)
                 proxyConf = (proxyHost, proxyPort, proxyAuth)
                 return self._TunnelingAgent(
                     reactor=reactor,
@@ -288,11 +338,11 @@ class ScrapyAgent:
                     bindAddress=bindaddress,
                     pool=self._pool,
                 )
-            proxyScheme = proxyScheme or b'http'
-            proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
+            proxyScheme = proxyScheme or b"http"
+            proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", ""))
             return self._ProxyAgent(
                 reactor=reactor,
-                proxyURI=to_bytes(proxyURI, encoding='ascii'),
+                proxyURI=to_bytes(proxyURI, encoding="ascii"),
                 connectTimeout=timeout,
                 bindAddress=bindaddress,
                 pool=self._pool,
@@ -308,7 +358,8 @@ class ScrapyAgent:
 
     def download_request(self, request):
         from twisted.internet import reactor
-        timeout = request.meta.get('download_timeout') or self._connectTimeout
+
+        timeout = request.meta.get("download_timeout") or self._connectTimeout
         agent = self._get_agent(request, timeout)
 
         # request details
@@ -316,13 +367,15 @@ class ScrapyAgent:
         method = to_bytes(request.method)
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
-            headers.removeHeader(b'Proxy-Authorization')
+            headers.removeHeader(b"Proxy-Authorization")
         if request.body:
             bodyproducer = _RequestBodyProducer(request.body)
         else:
             bodyproducer = None
         start_time = time()
-        d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
+        d = agent.request(
+            method, to_bytes(url, encoding="ascii"), headers, bodyproducer
+        )
         # set download latency
         d.addCallback(self._cb_latency, request, start_time)
         # response body is ready to be consumed
@@ -345,14 +398,14 @@ class ScrapyAgent:
         raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
 
     def _cb_latency(self, result, request, start_time):
-        request.meta['download_latency'] = time() - start_time
+        request.meta["download_latency"] = time() - start_time
         return result
 
     @staticmethod
     def _headers_from_twisted_response(response):
         headers = Headers()
         if response.length != UNKNOWN_LENGTH:
-            headers[b'Content-Length'] = str(response.length).encode()
+            headers[b"Content-Length"] = str(response.length).encode()
         headers.update(response.headers.getAllRawHeaders())
         return headers
 
@@ -366,8 +419,10 @@ class ScrapyAgent:
         )
         for handler, result in headers_received_result:
             if isinstance(result, Failure) and isinstance(result.value, StopDownload):
-                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
-                             {"request": request, "handler": handler.__qualname__})
+                logger.debug(
+                    "Download stopped for %(request)s from signal handler %(handler)s",
+                    {"request": request, "handler": handler.__qualname__},
+                )
                 txresponse._transport.stopProducing()
                 txresponse._transport.loseConnection()
                 return {
@@ -389,15 +444,23 @@ class ScrapyAgent:
                 "ip_address": None,
             }
 
-        maxsize = request.meta.get('download_maxsize', self._maxsize)
-        warnsize = request.meta.get('download_warnsize', self._warnsize)
+        maxsize = request.meta.get("download_maxsize", self._maxsize)
+        warnsize = request.meta.get("download_warnsize", self._warnsize)
         expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
-        fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
+        fail_on_dataloss = request.meta.get(
+            "download_fail_on_dataloss", self._fail_on_dataloss
+        )
 
         if maxsize and expected_size > maxsize:
-            warning_msg = ("Cancelling download of %(url)s: expected response "
-                           "size (%(size)s) larger than download max size (%(maxsize)s).")
-            warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+            warning_msg = (
+                "Cancelling download of %(url)s: expected response "
+                "size (%(size)s) larger than download max size (%(maxsize)s)."
+            )
+            warning_args = {
+                "url": request.url,
+                "size": expected_size,
+                "maxsize": maxsize,
+            }
 
             logger.warning(warning_msg, warning_args)
 
@@ -405,9 +468,11 @@ class ScrapyAgent:
             raise defer.CancelledError(warning_msg % warning_args)
 
         if warnsize and expected_size > warnsize:
-            logger.warning("Expected response size (%(size)s) larger than "
-                           "download warn size (%(warnsize)s) in request %(request)s.",
-                           {'size': expected_size, 'warnsize': warnsize, 'request': request})
+            logger.warning(
+                "Expected response size (%(size)s) larger than "
+                "download warn size (%(warnsize)s) in request %(request)s.",
+                {"size": expected_size, "warnsize": warnsize, "request": request},
+            )
 
         def _cancel(_):
             # Abort connection immediately.
@@ -457,7 +522,6 @@ class ScrapyAgent:
 
 @implementer(IBodyProducer)
 class _RequestBodyProducer:
-
     def __init__(self, body):
         self.body = body
         self.length = len(body)
@@ -474,8 +538,16 @@ class _RequestBodyProducer:
 
 
 class _ResponseReader(protocol.Protocol):
-
-    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
+    def __init__(
+        self,
+        finished,
+        txresponse,
+        request,
+        maxsize,
+        warnsize,
+        fail_on_dataloss,
+        crawler,
+    ):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -491,22 +563,28 @@ class _ResponseReader(protocol.Protocol):
         self._crawler = crawler
 
     def _finish_response(self, flags=None, failure=None):
-        self._finished.callback({
-            "txresponse": self._txresponse,
-            "body": self._bodybuf.getvalue(),
-            "flags": flags,
-            "certificate": self._certificate,
-            "ip_address": self._ip_address,
-            "failure": failure,
-        })
+        self._finished.callback(
+            {
+                "txresponse": self._txresponse,
+                "body": self._bodybuf.getvalue(),
+                "flags": flags,
+                "certificate": self._certificate,
+                "ip_address": self._ip_address,
+                "failure": failure,
+            }
+        )
 
     def connectionMade(self):
         if self._certificate is None:
             with suppress(AttributeError):
-                self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
+                self._certificate = ssl.Certificate(
+                    self.transport._producer.getPeerCertificate()
+                )
 
         if self._ip_address is None:
-            self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
+            self._ip_address = ipaddress.ip_address(
+                self.transport._producer.getPeer().host
+            )
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -524,29 +602,40 @@ class _ResponseReader(protocol.Protocol):
         )
         for handler, result in bytes_received_result:
             if isinstance(result, Failure) and isinstance(result.value, StopDownload):
-                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
-                             {"request": self._request, "handler": handler.__qualname__})
+                logger.debug(
+                    "Download stopped for %(request)s from signal handler %(handler)s",
+                    {"request": self._request, "handler": handler.__qualname__},
+                )
                 self.transport.stopProducing()
                 self.transport.loseConnection()
                 failure = result if result.value.fail else None
                 self._finish_response(flags=["download_stopped"], failure=failure)
 
         if self._maxsize and self._bytes_received > self._maxsize:
-            logger.warning("Received (%(bytes)s) bytes larger than download "
-                           "max size (%(maxsize)s) in request %(request)s.",
-                           {'bytes': self._bytes_received,
-                            'maxsize': self._maxsize,
-                            'request': self._request})
+            logger.warning(
+                "Received (%(bytes)s) bytes larger than download "
+                "max size (%(maxsize)s) in request %(request)s.",
+                {
+                    "bytes": self._bytes_received,
+                    "maxsize": self._maxsize,
+                    "request": self._request,
+                },
+            )
             # Clear buffer earlier to avoid keeping data in memory for a long time.
             self._bodybuf.truncate(0)
             self._finished.cancel()
 
-        if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
+        if (
+            self._warnsize
+            and self._bytes_received > self._warnsize
+            and not self._reached_warnsize
+        ):
             self._reached_warnsize = True
-            logger.warning("Received more bytes than download "
-                           "warn size (%(warnsize)s) in request %(request)s.",
-                           {'warnsize': self._warnsize,
-                            'request': self._request})
+            logger.warning(
+                "Received more bytes than download "
+                "warn size (%(warnsize)s) in request %(request)s.",
+                {"warnsize": self._warnsize, "request": self._request},
+            )
 
     def connectionLost(self, reason):
         if self._finished.called:
@@ -560,16 +649,20 @@ class _ResponseReader(protocol.Protocol):
             self._finish_response(flags=["partial"])
             return
 
-        if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
+        if reason.check(ResponseFailed) and any(
+            r.check(_DataLoss) for r in reason.value.reasons
+        ):
             if not self._fail_on_dataloss:
                 self._finish_response(flags=["dataloss"])
                 return
 
             if not self._fail_on_dataloss_warned:
-                logger.warning("Got data loss in %s. If you want to process broken "
-                               "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
-                               " -- This message won't be shown in further requests",
-                               self._txresponse.request.absoluteURI.decode())
+                logger.warning(
+                    "Got data loss in %s. If you want to process broken "
+                    "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
+                    " -- This message won't be shown in further requests",
+                    self._txresponse.request.absoluteURI.decode(),
+                )
                 self._fail_on_dataloss_warned = True
 
         self._finished.errback(reason)
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 3f1b36e92..25ac0307b 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -17,7 +17,9 @@ from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 
 
-H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
+H2DownloadHandlerOrSubclass = TypeVar(
+    "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
+)
 
 
 class H2DownloadHandler:
@@ -25,11 +27,14 @@ class H2DownloadHandler:
         self._crawler = crawler
 
         from twisted.internet import reactor
+
         self._pool = H2ConnectionPool(reactor, settings)
         self._context_factory = load_context_factory_from_settings(settings, crawler)
 
     @classmethod
-    def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
+    def from_crawler(
+        cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler
+    ) -> H2DownloadHandlerOrSubclass:
         return cls(crawler.settings, crawler)
 
     def download_request(self, request: Request, spider: Spider) -> Deferred:
@@ -49,7 +54,8 @@ class ScrapyH2Agent:
     _ProxyAgent = ScrapyProxyH2Agent
 
     def __init__(
-        self, context_factory,
+        self,
+        context_factory,
         pool: H2ConnectionPool,
         connect_timeout: int = 10,
         bind_address: Optional[bytes] = None,
@@ -63,19 +69,22 @@ class ScrapyH2Agent:
 
     def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
         from twisted.internet import reactor
-        bind_address = request.meta.get('bindaddress') or self._bind_address
-        proxy = request.meta.get('proxy')
+
+        bind_address = request.meta.get("bindaddress") or self._bind_address
+        proxy = request.meta.get("proxy")
         if proxy:
             _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
             scheme = _parse(request.url)[0]
 
-            if scheme == b'https':
+            if scheme == b"https":
                 # ToDo
-                raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
+                raise NotImplementedError(
+                    "Tunneling via CONNECT method using HTTP/2.0 is not yet supported"
+                )
             return self._ProxyAgent(
                 reactor=reactor,
                 context_factory=self._context_factory,
-                proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
+                proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),
                 connect_timeout=timeout,
                 bind_address=bind_address,
                 pool=self._pool,
@@ -91,7 +100,8 @@ class ScrapyH2Agent:
 
     def download_request(self, request: Request, spider: Spider) -> Deferred:
         from twisted.internet import reactor
-        timeout = request.meta.get('download_timeout') or self._connect_timeout
+
+        timeout = request.meta.get("download_timeout") or self._connect_timeout
         agent = self._get_agent(request, timeout)
 
         start_time = time()
@@ -103,12 +113,16 @@ class ScrapyH2Agent:
         return d
 
     @staticmethod
-    def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
-        request.meta['download_latency'] = time() - start_time
+    def _cb_latency(
+        response: Response, request: Request, start_time: float
+    ) -> Response:
+        request.meta["download_latency"] = time() - start_time
         return response
 
     @staticmethod
-    def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
+    def _cb_timeout(
+        response: Response, request: Request, timeout: float, timeout_cl: DelayedCall
+    ) -> Response:
         if timeout_cl.active():
             timeout_cl.cancel()
             return response
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 51ca1ed5e..81d8e8115 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -6,40 +6,49 @@ from scrapy.utils.misc import create_instance
 
 
 class S3DownloadHandler:
-
-    def __init__(self, settings, *,
-                 crawler=None,
-                 aws_access_key_id=None, aws_secret_access_key=None,
-                 aws_session_token=None,
-                 httpdownloadhandler=HTTPDownloadHandler, **kw):
+    def __init__(
+        self,
+        settings,
+        *,
+        crawler=None,
+        aws_access_key_id=None,
+        aws_secret_access_key=None,
+        aws_session_token=None,
+        httpdownloadhandler=HTTPDownloadHandler,
+        **kw,
+    ):
         if not is_botocore_available():
-            raise NotConfigured('missing botocore library')
+            raise NotConfigured("missing botocore library")
 
         if not aws_access_key_id:
-            aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
+            aws_access_key_id = settings["AWS_ACCESS_KEY_ID"]
         if not aws_secret_access_key:
-            aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
+            aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"]
         if not aws_session_token:
-            aws_session_token = settings['AWS_SESSION_TOKEN']
+            aws_session_token = settings["AWS_SESSION_TOKEN"]
 
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
         # unless 'anon' was set explicitly (True/False).
-        anon = kw.get('anon')
+        anon = kw.get("anon")
         if anon is None and not aws_access_key_id and not aws_secret_access_key:
-            kw['anon'] = True
-        self.anon = kw.get('anon')
+            kw["anon"] = True
+        self.anon = kw.get("anon")
 
         self._signer = None
         import botocore.auth
         import botocore.credentials
-        kw.pop('anon', None)
+
+        kw.pop("anon", None)
         if kw:
-            raise TypeError(f'Unexpected keyword arguments: {kw}')
+            raise TypeError(f"Unexpected keyword arguments: {kw}")
         if not self.anon:
-            SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
-            self._signer = SignerCls(botocore.credentials.Credentials(
-                aws_access_key_id, aws_secret_access_key, aws_session_token))
+            SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
+            self._signer = SignerCls(
+                botocore.credentials.Credentials(
+                    aws_access_key_id, aws_secret_access_key, aws_session_token
+                )
+            )
 
         _http_handler = create_instance(
             objcls=httpdownloadhandler,
@@ -54,20 +63,21 @@ class S3DownloadHandler:
 
     def download_request(self, request, spider):
         p = urlparse_cached(request)
-        scheme = 'https' if request.meta.get('is_secure') else 'http'
+        scheme = "https" if request.meta.get("is_secure") else "http"
         bucket = p.hostname
-        path = p.path + '?' + p.query if p.query else p.path
-        url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
+        path = p.path + "?" + p.query if p.query else p.path
+        url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
         if self.anon:
             request = request.replace(url=url)
         else:
             import botocore.awsrequest
+
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
-                url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
+                url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
                 headers=request.headers.to_unicode_dict(),
-                data=request.body)
+                data=request.body,
+            )
             self._signer.add_auth(awsrequest)
-            request = request.replace(
-                url=url, headers=awsrequest.headers.items())
+            request = request.replace(url=url, headers=awsrequest.headers.items())
         return self._download_http(request, spider)
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 289147466..3410b4255 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -18,28 +18,31 @@ from scrapy.utils.conf import build_component_list
 
 class DownloaderMiddlewareManager(MiddlewareManager):
 
-    component_name = 'downloader middleware'
+    component_name = "downloader middleware"
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(
-            settings.getwithbase('DOWNLOADER_MIDDLEWARES'))
+        return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES"))
 
     def _add_middleware(self, mw):
-        if hasattr(mw, 'process_request'):
-            self.methods['process_request'].append(mw.process_request)
-        if hasattr(mw, 'process_response'):
-            self.methods['process_response'].appendleft(mw.process_response)
-        if hasattr(mw, 'process_exception'):
-            self.methods['process_exception'].appendleft(mw.process_exception)
+        if hasattr(mw, "process_request"):
+            self.methods["process_request"].append(mw.process_request)
+        if hasattr(mw, "process_response"):
+            self.methods["process_response"].appendleft(mw.process_response)
+        if hasattr(mw, "process_exception"):
+            self.methods["process_exception"].appendleft(mw.process_exception)
 
     def download(self, download_func: Callable, request: Request, spider: Spider):
         @defer.inlineCallbacks
         def process_request(request: Request):
-            for method in self.methods['process_request']:
+            for method in self.methods["process_request"]:
                 method = cast(Callable, method)
-                response = yield deferred_from_coro(method(request=request, spider=spider))
-                if response is not None and not isinstance(response, (Response, Request)):
+                response = yield deferred_from_coro(
+                    method(request=request, spider=spider)
+                )
+                if response is not None and not isinstance(
+                    response, (Response, Request)
+                ):
                     raise _InvalidOutput(
                         f"Middleware {method.__qualname__} must return None, Response or "
                         f"Request, got {response.__class__.__name__}"
@@ -55,9 +58,11 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             elif isinstance(response, Request):
                 return response
 
-            for method in self.methods['process_response']:
+            for method in self.methods["process_response"]:
                 method = cast(Callable, method)
-                response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
+                response = yield deferred_from_coro(
+                    method(request=request, response=response, spider=spider)
+                )
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
                         f"Middleware {method.__qualname__} must return Response or Request, "
@@ -70,10 +75,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         @defer.inlineCallbacks
         def process_exception(failure: Failure):
             exception = failure.value
-            for method in self.methods['process_exception']:
+            for method in self.methods["process_exception"]:
                 method = cast(Callable, method)
-                response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
-                if response is not None and not isinstance(response, (Response, Request)):
+                response = yield deferred_from_coro(
+                    method(request=request, exception=exception, spider=spider)
+                )
+                if response is not None and not isinstance(
+                    response, (Response, Request)
+                ):
                     raise _InvalidOutput(
                         f"Middleware {method.__qualname__} must return None, Response or "
                         f"Request, got {type(response)}"
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 7d67a426f..db2639a75 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -2,7 +2,11 @@ import logging
 
 from OpenSSL import SSL
 from service_identity.exceptions import CertificateError
-from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
+from twisted.internet._sslverify import (
+    ClientTLSOptions,
+    verifyHostname,
+    VerificationError,
+)
 from twisted.internet.ssl import AcceptableCiphers
 
 from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@@ -10,17 +14,17 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 logger = logging.getLogger(__name__)
 
 
-METHOD_TLS = 'TLS'
-METHOD_TLSv10 = 'TLSv1.0'
-METHOD_TLSv11 = 'TLSv1.1'
-METHOD_TLSv12 = 'TLSv1.2'
+METHOD_TLS = "TLS"
+METHOD_TLSv10 = "TLSv1.0"
+METHOD_TLSv11 = "TLSv1.1"
+METHOD_TLSv12 = "TLSv1.2"
 
 
 openssl_methods = {
-    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
-    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
-    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
+    METHOD_TLS: SSL.SSLv23_METHOD,  # protocol negotiation (recommended)
+    METHOD_TLSv10: SSL.TLSv1_METHOD,  # TLS 1.0 only
+    METHOD_TLSv11: getattr(SSL, "TLSv1_1_METHOD", 5),  # TLS 1.1 only
+    METHOD_TLSv12: getattr(SSL, "TLSv1_2_METHOD", 6),  # TLS 1.2 only
 }
 
 
@@ -44,32 +48,38 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
             connection.set_tlsext_host_name(self._hostnameBytes)
         elif where & SSL.SSL_CB_HANDSHAKE_DONE:
             if self.verbose_logging:
-                logger.debug('SSL connection to %s using protocol %s, cipher %s',
-                             self._hostnameASCII,
-                             connection.get_protocol_version_name(),
-                             connection.get_cipher_name(),
-                             )
+                logger.debug(
+                    "SSL connection to %s using protocol %s, cipher %s",
+                    self._hostnameASCII,
+                    connection.get_protocol_version_name(),
+                    connection.get_cipher_name(),
+                )
                 server_cert = connection.get_peer_certificate()
-                logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
-                             x509name_to_string(server_cert.get_issuer()),
-                             x509name_to_string(server_cert.get_subject()),
-                             )
+                logger.debug(
+                    'SSL connection certificate: issuer "%s", subject "%s"',
+                    x509name_to_string(server_cert.get_issuer()),
+                    x509name_to_string(server_cert.get_subject()),
+                )
                 key_info = get_temp_key_info(connection._ssl)
                 if key_info:
-                    logger.debug('SSL temp key: %s', key_info)
+                    logger.debug("SSL temp key: %s", key_info)
 
             try:
                 verifyHostname(connection, self._hostnameASCII)
             except (CertificateError, VerificationError) as e:
                 logger.warning(
                     'Remote certificate is not valid for hostname "%s"; %s',
-                    self._hostnameASCII, e)
+                    self._hostnameASCII,
+                    e,
+                )
 
             except ValueError as e:
                 logger.warning(
-                    'Ignoring error while verifying certificate '
+                    "Ignoring error while verifying certificate "
                     'from host "%s" (exception: %r)',
-                    self._hostnameASCII, e)
+                    self._hostnameASCII,
+                    e,
+                )
 
 
-DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 255ca62e6..a00afb7af 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -15,33 +15,33 @@ from scrapy.responsetypes import responsetypes
 def _parsed_url_args(parsed):
     # Assume parsed is urlparse-d from Request.url,
     # which was passed via safe_url_string and is ascii-only.
-    path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
+    path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, ""))
     path = to_bytes(path, encoding="ascii")
     host = to_bytes(parsed.hostname, encoding="ascii")
     port = parsed.port
     scheme = to_bytes(parsed.scheme, encoding="ascii")
     netloc = to_bytes(parsed.netloc, encoding="ascii")
     if port is None:
-        port = 443 if scheme == b'https' else 80
+        port = 443 if scheme == b"https" else 80
     return scheme, netloc, host, port, path
 
 
 def _parse(url):
-    """ Return tuple of (scheme, netloc, host, port, path),
+    """Return tuple of (scheme, netloc, host, port, path),
     all in bytes except for port which is int.
     Assume url is from Request.url, which was passed via safe_url_string
     and is ascii-only.
     """
     url = url.strip()
-    if not re.match(r'^\w+://', url):
-        url = '//' + url
+    if not re.match(r"^\w+://", url):
+        url = "//" + url
     parsed = urlparse(url)
     return _parsed_url_args(parsed)
 
 
 class ScrapyHTTPPageGetter(HTTPClient):
 
-    delimiter = b'\n'
+    delimiter = b"\n"
 
     def connectionMade(self):
         self.headers = Headers()  # bucket for response headers
@@ -75,8 +75,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.factory.noPage(reason)
 
     def handleResponse(self, response):
-        if self.factory.method.upper() == b'HEAD':
-            self.factory.page(b'')
+        if self.factory.method.upper() == b"HEAD":
+            self.factory.page(b"")
         elif self.length is not None and self.length > 0:
             self.factory.noPage(self._connection_lost_reason)
         else:
@@ -87,12 +87,15 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.transport.loseConnection()
 
         # transport cleanup needed for HTTPS connections
-        if self.factory.url.startswith(b'https'):
+        if self.factory.url.startswith(b"https"):
             self.transport.stopProducing()
 
         self.factory.noPage(
-            defer.TimeoutError(f"Getting {self.factory.url} took longer "
-                               f"than {self.factory.timeout} seconds."))
+            defer.TimeoutError(
+                f"Getting {self.factory.url} took longer "
+                f"than {self.factory.timeout} seconds."
+            )
+        )
 
 
 # This class used to inherit from Twisted’s
@@ -109,16 +112,24 @@ class ScrapyHTTPClientFactory(ClientFactory):
     afterFoundGet = False
 
     def _build_response(self, body, request):
-        request.meta['download_latency'] = self.headers_time - self.start_time
+        request.meta["download_latency"] = self.headers_time - self.start_time
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
-        return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
+        return respcls(
+            url=self._url,
+            status=status,
+            headers=headers,
+            body=body,
+            protocol=to_unicode(self.version),
+        )
 
     def _set_connection_attributes(self, request):
         parsed = urlparse_cached(request)
-        self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
-        proxy = request.meta.get('proxy')
+        self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(
+            parsed
+        )
+        proxy = request.meta.get("proxy")
         if proxy:
             self.scheme, _, self.host, self.port, _ = _parse(proxy)
             self.path = self.url
@@ -126,12 +137,12 @@ class ScrapyHTTPClientFactory(ClientFactory):
     def __init__(self, request, timeout=180):
         self._url = urldefrag(request.url)[0]
         # converting to bytes to comply to Twisted interface
-        self.url = to_bytes(self._url, encoding='ascii')
-        self.method = to_bytes(request.method, encoding='ascii')
+        self.url = to_bytes(self._url, encoding="ascii")
+        self.method = to_bytes(request.method, encoding="ascii")
         self.body = request.body or None
         self.headers = Headers(request.headers)
         self.response_headers = None
-        self.timeout = request.meta.get('download_timeout') or timeout
+        self.timeout = request.meta.get("download_timeout") or timeout
         self.start_time = time()
         self.deferred = defer.Deferred().addCallback(self._build_response, request)
 
@@ -146,16 +157,16 @@ class ScrapyHTTPClientFactory(ClientFactory):
         self._set_connection_attributes(request)
 
         # set Host header based on url
-        self.headers.setdefault('Host', self.netloc)
+        self.headers.setdefault("Host", self.netloc)
 
         # set Content-Length based len of body
         if self.body is not None:
-            self.headers['Content-Length'] = len(self.body)
+            self.headers["Content-Length"] = len(self.body)
             # just in case a broken http/1.1 decides to keep connection alive
             self.headers.setdefault("Connection", "close")
         # Content-Length must be specified in POST method even with no body
-        elif self.method == b'POST':
-            self.headers['Content-Length'] = 0
+        elif self.method == b"POST":
+            self.headers["Content-Length"] = 0
 
     def __repr__(self):
         return f"<{self.__class__.__name__}: {self.url}>"
@@ -171,6 +182,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         p.afterFoundGet = self.afterFoundGet
         if self.timeout:
             from twisted.internet import reactor
+
             timeoutCall = reactor.callLater(self.timeout, p.timeout)
             self.deferred.addBoth(self._cancelTimeout, timeoutCall)
         return p
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 1228e78da..19696415b 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -79,13 +79,14 @@ class ExecutionEngine:
         self.running = False
         self.paused = False
         self.scheduler_cls = self._get_scheduler_class(crawler.settings)
-        downloader_cls = load_object(self.settings['DOWNLOADER'])
+        downloader_cls = load_object(self.settings["DOWNLOADER"])
         self.downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
         self._spider_closed_callback = spider_closed_callback
 
     def _get_scheduler_class(self, settings: BaseSettings) -> type:
         from scrapy.core.scheduler import BaseScheduler
+
         scheduler_cls = load_object(settings["SCHEDULER"])
         if not issubclass(scheduler_cls, BaseScheduler):
             raise TypeError(
@@ -106,6 +107,7 @@ class ExecutionEngine:
 
     def stop(self) -> Deferred:
         """Gracefully stop the execution engine"""
+
         @inlineCallbacks
         def _finish_stopping_engine(_) -> Deferred:
             yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
@@ -115,7 +117,11 @@ class ExecutionEngine:
             raise RuntimeError("Engine not running")
 
         self.running = False
-        dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None)
+        dfd = (
+            self.close_spider(self.spider, reason="shutdown")
+            if self.spider is not None
+            else succeed(None)
+        )
         return dfd.addBoth(_finish_stopping_engine)
 
     def close(self) -> Deferred:
@@ -126,7 +132,9 @@ class ExecutionEngine:
         if self.running:
             return self.stop()  # will also close spider and downloader
         if self.spider is not None:
-            return self.close_spider(self.spider, reason="shutdown")  # will also close downloader
+            return self.close_spider(
+                self.spider, reason="shutdown"
+            )  # will also close downloader
         return succeed(self.downloader.close())
 
     def pause(self) -> None:
@@ -144,7 +152,10 @@ class ExecutionEngine:
         if self.paused:
             return None
 
-        while not self._needs_backout() and self._next_request_from_scheduler() is not None:
+        while (
+            not self._needs_backout()
+            and self._next_request_from_scheduler() is not None
+        ):
             pass
 
         if self.slot.start_requests is not None and not self._needs_backout():
@@ -154,7 +165,11 @@ class ExecutionEngine:
                 self.slot.start_requests = None
             except Exception:
                 self.slot.start_requests = None
-                logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider})
+                logger.error(
+                    "Error while obtaining start requests",
+                    exc_info=True,
+                    extra={"spider": self.spider},
+                )
             else:
                 self.crawl(request)
 
@@ -179,18 +194,30 @@ class ExecutionEngine:
 
         d = self._download(request, self.spider)
         d.addBoth(self._handle_downloader_output, request)
-        d.addErrback(lambda f: logger.info('Error while handling downloader output',
-                                           exc_info=failure_to_exc_info(f),
-                                           extra={'spider': self.spider}))
+        d.addErrback(
+            lambda f: logger.info(
+                "Error while handling downloader output",
+                exc_info=failure_to_exc_info(f),
+                extra={"spider": self.spider},
+            )
+        )
         d.addBoth(lambda _: self.slot.remove_request(request))
-        d.addErrback(lambda f: logger.info('Error while removing request from slot',
-                                           exc_info=failure_to_exc_info(f),
-                                           extra={'spider': self.spider}))
+        d.addErrback(
+            lambda f: logger.info(
+                "Error while removing request from slot",
+                exc_info=failure_to_exc_info(f),
+                extra={"spider": self.spider},
+            )
+        )
         slot = self.slot
         d.addBoth(lambda _: slot.nextcall.schedule())
-        d.addErrback(lambda f: logger.info('Error while scheduling new request',
-                                           exc_info=failure_to_exc_info(f),
-                                           extra={'spider': self.spider}))
+        d.addErrback(
+            lambda f: logger.info(
+                "Error while scheduling new request",
+                exc_info=failure_to_exc_info(f),
+                extra={"spider": self.spider},
+            )
+        )
         return d
 
     def _handle_downloader_output(
@@ -199,7 +226,9 @@ class ExecutionEngine:
         assert self.spider is not None  # typing
 
         if not isinstance(result, (Request, Response, Failure)):
-            raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
+            raise TypeError(
+                f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}"
+            )
 
         # downloader middleware can return requests (for example, redirects)
         if isinstance(result, Request):
@@ -211,7 +240,7 @@ class ExecutionEngine:
             lambda f: logger.error(
                 "Error while enqueuing downloader output",
                 exc_info=failure_to_exc_info(f),
-                extra={'spider': self.spider},
+                extra={"spider": self.spider},
             )
         )
         return d
@@ -244,16 +273,22 @@ class ExecutionEngine:
                 stacklevel=2,
             )
             if spider is not self.spider:
-                raise RuntimeError(f"The spider {spider.name!r} does not match the open spider")
+                raise RuntimeError(
+                    f"The spider {spider.name!r} does not match the open spider"
+                )
         if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
         self._schedule_request(request, self.spider)
         self.slot.nextcall.schedule()  # type: ignore[union-attr]
 
     def _schedule_request(self, request: Request, spider: Spider) -> None:
-        self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
+        self.signals.send_catch_log(
+            signals.request_scheduled, request=request, spider=spider
+        )
         if not self.slot.scheduler.enqueue_request(request):  # type: ignore[union-attr]
-            self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
+            self.signals.send_catch_log(
+                signals.request_dropped, request=request, spider=spider
+            )
 
     def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
         """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
@@ -264,10 +299,14 @@ class ExecutionEngine:
                 stacklevel=2,
             )
             if spider is not self.spider:
-                logger.warning("The spider '%s' does not match the open spider", spider.name)
+                logger.warning(
+                    "The spider '%s' does not match the open spider", spider.name
+                )
         if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
-        return self._download(request, spider).addBoth(self._downloaded, request, spider)
+        return self._download(request, spider).addBoth(
+            self._downloaded, request, spider
+        )
 
     def _downloaded(
         self, result: Union[Response, Request], request: Request, spider: Spider
@@ -286,7 +325,9 @@ class ExecutionEngine:
 
         def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
             if not isinstance(result, (Response, Request)):
-                raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
+                raise TypeError(
+                    f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}"
+                )
             if isinstance(result, Response):
                 if result.request is None:
                     result.request = request
@@ -311,13 +352,19 @@ class ExecutionEngine:
         return dwld
 
     @inlineCallbacks
-    def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True):
+    def open_spider(
+        self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
+    ):
         if self.slot is not None:
             raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
-        logger.info("Spider opened", extra={'spider': spider})
+        logger.info("Spider opened", extra={"spider": spider})
         nextcall = CallLaterOnce(self._next_request)
-        scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler)
-        start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
+        scheduler = create_instance(
+            self.scheduler_cls, settings=None, crawler=self.crawler
+        )
+        start_requests = yield self.scraper.spidermw.process_start_requests(
+            start_requests, spider
+        )
         self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
         self.spider = spider
         if hasattr(scheduler, "open"):
@@ -337,7 +384,9 @@ class ExecutionEngine:
         """
         assert self.spider is not None  # typing
         expected_ex = (DontCloseSpider, CloseSpider)
-        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
+        res = self.signals.send_catch_log(
+            signals.spider_idle, spider=self.spider, dont_log=expected_ex
+        )
         detected_ex = {
             ex: x.value
             for _, x in res
@@ -347,7 +396,7 @@ class ExecutionEngine:
         if DontCloseSpider in detected_ex:
             return None
         if self.spider_is_idle():
-            ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished'))
+            ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished"))
             assert isinstance(ex, CloseSpider)  # typing
             self.close_spider(self.spider, reason=ex.reason)
 
@@ -359,40 +408,55 @@ class ExecutionEngine:
         if self.slot.closing is not None:
             return self.slot.closing
 
-        logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})
+        logger.info(
+            "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
+        )
 
         dfd = self.slot.close()
 
         def log_failure(msg: str) -> Callable:
             def errback(failure: Failure) -> None:
-                logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
+                logger.error(
+                    msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider}
+                )
+
             return errback
 
         dfd.addBoth(lambda _: self.downloader.close())
-        dfd.addErrback(log_failure('Downloader close failure'))
+        dfd.addErrback(log_failure("Downloader close failure"))
 
         dfd.addBoth(lambda _: self.scraper.close_spider(spider))
-        dfd.addErrback(log_failure('Scraper close failure'))
+        dfd.addErrback(log_failure("Scraper close failure"))
 
         if hasattr(self.slot.scheduler, "close"):
             dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
             dfd.addErrback(log_failure("Scheduler close failure"))
 
-        dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
-            signal=signals.spider_closed, spider=spider, reason=reason,
-        ))
-        dfd.addErrback(log_failure('Error while sending spider_close signal'))
+        dfd.addBoth(
+            lambda _: self.signals.send_catch_log_deferred(
+                signal=signals.spider_closed,
+                spider=spider,
+                reason=reason,
+            )
+        )
+        dfd.addErrback(log_failure("Error while sending spider_close signal"))
 
         dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
-        dfd.addErrback(log_failure('Stats close failure'))
+        dfd.addErrback(log_failure("Stats close failure"))
 
-        dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))
+        dfd.addBoth(
+            lambda _: logger.info(
+                "Spider closed (%(reason)s)",
+                {"reason": reason},
+                extra={"spider": spider},
+            )
+        )
 
-        dfd.addBoth(lambda _: setattr(self, 'slot', None))
-        dfd.addErrback(log_failure('Error while unassigning slot'))
+        dfd.addBoth(lambda _: setattr(self, "slot", None))
+        dfd.addErrback(log_failure("Error while unassigning slot"))
 
-        dfd.addBoth(lambda _: setattr(self, 'spider', None))
-        dfd.addErrback(log_failure('Error while unassigning spider'))
+        dfd.addBoth(lambda _: setattr(self, "spider", None))
+        dfd.addErrback(log_failure("Error while unassigning spider"))
 
         dfd.addBoth(lambda _: self._spider_closed_callback(spider))
 
@@ -408,7 +472,11 @@ class ExecutionEngine:
         return [self.spider] if self.spider is not None else []
 
     def has_capacity(self) -> bool:
-        warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2)
+        warnings.warn(
+            "ExecutionEngine.has_capacity is deprecated",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         return not bool(self.slot)
 
     def schedule(self, request: Request, spider: Spider) -> None:
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index c6b357be3..119443c80 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -28,7 +28,9 @@ class H2ConnectionPool:
         # Save all requests that arrive before the connection is established
         self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
 
-    def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
+    def get_connection(
+        self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
+    ) -> Deferred:
         if key in self._pending_requests:
             # Received a request while connecting to remote
             # Create a deferred which will fire with the H2ClientProtocol
@@ -46,7 +48,9 @@ class H2ConnectionPool:
         # No connection is established for the given URI
         return self._new_connection(key, uri, endpoint)
 
-    def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
+    def _new_connection(
+        self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
+    ) -> Deferred:
         self._pending_requests[key] = deque()
 
         conn_lost_deferred = Deferred()
@@ -102,7 +106,9 @@ class H2Agent:
     ) -> None:
         self._reactor = reactor
         self._pool = pool
-        self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
+        self._context_factory = AcceptableProtocolsContextFactory(
+            context_factory, acceptable_protocols=[b"h2"]
+        )
         self.endpoint_factory = _StandardEndpointFactory(
             self._reactor, self._context_factory, connect_timeout, bind_address
         )
@@ -118,7 +124,7 @@ class H2Agent:
         return uri.scheme, uri.host, uri.port
 
     def request(self, request: Request, spider: Spider) -> Deferred:
-        uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
+        uri = URI.fromBytes(bytes(request.url, encoding="utf-8"))
         try:
             endpoint = self.get_endpoint(uri)
         except SchemeNotSupported:
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 6047f9ca8..214deeed0 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -9,9 +9,15 @@ from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.errors import ErrorCodes
 from h2.events import (
-    Event, ConnectionTerminated, DataReceived, ResponseReceived,
-    SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
-    WindowUpdated
+    Event,
+    ConnectionTerminated,
+    DataReceived,
+    ResponseReceived,
+    SettingsAcknowledged,
+    StreamEnded,
+    StreamReset,
+    UnknownFrameReceived,
+    WindowUpdated,
 )
 from h2.exceptions import FrameTooLargeError, H2Error
 from twisted.internet.defer import Deferred
@@ -37,7 +43,6 @@ PROTOCOL_NAME = b"h2"
 
 
 class InvalidNegotiatedProtocol(H2Error):
-
     def __init__(self, negotiated_protocol: bytes) -> None:
         self.negotiated_protocol = negotiated_protocol
 
@@ -55,11 +60,13 @@ class RemoteTerminatedConnection(H2Error):
         self.terminate_event = event
 
     def __str__(self) -> str:
-        return f'Received GOAWAY frame from {self.remote_ip_address!r}'
+        return f"Received GOAWAY frame from {self.remote_ip_address!r}"
 
 
 class MethodNotAllowed405(H2Error):
-    def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
+    def __init__(
+        self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]
+    ) -> None:
         self.remote_ip_address = remote_ip_address
 
     def __str__(self) -> str:
@@ -70,7 +77,9 @@ class MethodNotAllowed405(H2Error):
 class H2ClientProtocol(Protocol, TimeoutMixin):
     IDLE_TIMEOUT = 240
 
-    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
+    def __init__(
+        self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
+    ) -> None:
         """
         Arguments:
             uri -- URI of the base url to which HTTP/2 Connection will be made.
@@ -82,7 +91,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """
         self._conn_lost_deferred = conn_lost_deferred
 
-        config = H2Configuration(client_side=True, header_encoding='utf-8')
+        config = H2Configuration(client_side=True, header_encoding="utf-8")
         self.conn = H2Connection(config=config)
 
         # ID of the next request stream
@@ -105,31 +114,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         # initialized when connection is successfully made
         self.metadata: Dict = {
             # Peer certificate instance
-            'certificate': None,
-
+            "certificate": None,
             # Address of the server we are connected to which
             # is updated when HTTP/2 connection is  made successfully
-            'ip_address': None,
-
+            "ip_address": None,
             # URI of the peer HTTP/2 connection is made
-            'uri': uri,
-
+            "uri": uri,
             # Both ip_address and uri are used by the Stream before
             # initiating the request to verify that the base address
-
             # Variables taken from Project Settings
-            'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
-            'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
-
+            "default_download_maxsize": settings.getint("DOWNLOAD_MAXSIZE"),
+            "default_download_warnsize": settings.getint("DOWNLOAD_WARNSIZE"),
             # Counter to keep track of opened streams. This counter
             # is used to make sure that not more than MAX_CONCURRENT_STREAMS
             # streams are opened which leads to ProtocolError
             # We use simple FIFO policy to handle pending requests
-            'active_streams': 0,
-
+            "active_streams": 0,
             # Flag to keep track if settings were acknowledged by the remote
             # This ensures that we have established a HTTP/2 connection
-            'settings_acknowledged': False,
+            "settings_acknowledged": False,
         }
 
     @property
@@ -138,7 +141,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         This is used while initiating pending streams to make sure
         that we initiate stream only during active HTTP/2 Connection
         """
-        return bool(self.transport.connected) and self.metadata['settings_acknowledged']
+        return bool(self.transport.connected) and self.metadata["settings_acknowledged"]
 
     @property
     def allowed_max_concurrent_streams(self) -> int:
@@ -149,7 +152,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """
         return min(
             self.conn.local_settings.max_concurrent_streams,
-            self.conn.remote_settings.max_concurrent_streams
+            self.conn.remote_settings.max_concurrent_streams,
         )
 
     def _send_pending_requests(self) -> None:
@@ -159,37 +162,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """
         while (
             self._pending_request_stream_pool
-            and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
+            and self.metadata["active_streams"] < self.allowed_max_concurrent_streams
             and self.h2_connected
         ):
-            self.metadata['active_streams'] += 1
+            self.metadata["active_streams"] += 1
             stream = self._pending_request_stream_pool.popleft()
             stream.initiate_request()
             self._write_to_transport()
 
     def pop_stream(self, stream_id: int) -> Stream:
-        """Perform cleanup when a stream is closed
-        """
+        """Perform cleanup when a stream is closed"""
         stream = self.streams.pop(stream_id)
-        self.metadata['active_streams'] -= 1
+        self.metadata["active_streams"] -= 1
         self._send_pending_requests()
         return stream
 
     def _new_stream(self, request: Request, spider: Spider) -> Stream:
-        """Instantiates a new Stream object
-        """
+        """Instantiates a new Stream object"""
         stream = Stream(
             stream_id=next(self._stream_id_generator),
             request=request,
             protocol=self,
-            download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']),
-            download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']),
+            download_maxsize=getattr(
+                spider, "download_maxsize", self.metadata["default_download_maxsize"]
+            ),
+            download_warnsize=getattr(
+                spider, "download_warnsize", self.metadata["default_download_warnsize"]
+            ),
         )
         self.streams[stream.stream_id] = stream
         return stream
 
     def _write_to_transport(self) -> None:
-        """ Write data to the underlying transport connection
+        """Write data to the underlying transport connection
         from the HTTP2 connection instance if any
         """
         # Reset the idle timeout as connection is still actively sending data
@@ -200,7 +205,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
     def request(self, request: Request, spider: Spider) -> Deferred:
         if not isinstance(request, Request):
-            raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
+            raise TypeError(
+                f"Expected scrapy.http.Request, received {request.__class__.__qualname__}"
+            )
 
         stream = self._new_stream(request, spider)
         d = stream.get_response()
@@ -221,7 +228,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.setTimeout(self.IDLE_TIMEOUT)
 
         destination = self.transport.getPeer()
-        self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
+        self.metadata["ip_address"] = ipaddress.ip_address(destination.host)
 
         # Initiate H2 Connection
         self.conn.initiate_connection()
@@ -237,9 +244,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """
         Close the connection if it's not made via the expected protocol
         """
-        if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME:
+        if (
+            self.transport.negotiatedProtocol is not None
+            and self.transport.negotiatedProtocol != PROTOCOL_NAME
+        ):
             # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer
-            self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)])
+            self._lose_connection_with_error(
+                [InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]
+            )
 
     def _check_received_data(self, data: bytes) -> None:
         """Checks for edge cases where the connection to remote fails
@@ -248,8 +260,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         Arguments:
             data -- Data received from the remote
         """
-        if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):
-            raise MethodNotAllowed405(self.metadata['ip_address'])
+        if data.startswith(b"HTTP/2.0 405 Method Not Allowed"):
+            raise MethodNotAllowed405(self.metadata["ip_address"])
 
     def dataReceived(self, data: bytes) -> None:
         # Reset the idle timeout as connection is still actively receiving data
@@ -284,7 +296,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         if (
             self.conn.open_outbound_streams > 0
             or self.conn.open_inbound_streams > 0
-            or self.metadata['active_streams'] > 0
+            or self.metadata["active_streams"] > 0
         ):
             error_code = ErrorCodes.PROTOCOL_ERROR
         else:
@@ -292,9 +304,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.conn.close_connection(error_code=error_code)
         self._write_to_transport()
 
-        self._lose_connection_with_error([
-            TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")
-        ])
+        self._lose_connection_with_error(
+            [TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
+        )
 
     def connectionLost(self, reason: Failure = connectionDone) -> None:
         """Called by Twisted when the transport connection is lost.
@@ -311,13 +323,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self._conn_lost_deferred.callback(self._conn_lost_errors)
 
         for stream in self.streams.values():
-            if stream.metadata['request_sent']:
+            if stream.metadata["request_sent"]:
                 close_reason = StreamCloseReason.CONNECTION_LOST
             else:
                 close_reason = StreamCloseReason.INACTIVE
             stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
 
-        self.metadata['active_streams'] -= len(self.streams)
+        self.metadata["active_streams"] -= len(self.streams)
         self.streams.clear()
         self._pending_request_stream_pool.clear()
         self.conn.close_connection()
@@ -345,13 +357,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             elif isinstance(event, UnknownFrameReceived):
-                logger.warning('Unknown frame received: %s', event.frame)
+                logger.warning("Unknown frame received: %s", event.frame)
 
     # Event handler functions starts here
     def connection_terminated(self, event: ConnectionTerminated) -> None:
-        self._lose_connection_with_error([
-            RemoteTerminatedConnection(self.metadata['ip_address'], event)
-        ])
+        self._lose_connection_with_error(
+            [RemoteTerminatedConnection(self.metadata["ip_address"], event)]
+        )
 
     def data_received(self, event: DataReceived) -> None:
         try:
@@ -370,14 +382,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             stream.receive_headers(event.headers)
 
     def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
-        self.metadata['settings_acknowledged'] = True
+        self.metadata["settings_acknowledged"] = True
 
         # Send off all the pending requests as now we have
         # established a proper HTTP/2 connection
         self._send_pending_requests()
 
         # Update certificate when our HTTP/2 connection is established
-        self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
+        self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate())
 
     def stream_ended(self, event: StreamEnded) -> None:
         try:
@@ -406,7 +418,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
 @implementer(IProtocolNegotiationFactory)
 class H2ClientFactory(Factory):
-    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
+    def __init__(
+        self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
+    ) -> None:
         self.uri = uri
         self.settings = settings
         self.conn_lost_deferred = conn_lost_deferred
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index e197790f7..1b185cd8c 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -32,18 +32,19 @@ class InactiveStreamClosed(ConnectionClosed):
         self.request = request
 
     def __str__(self) -> str:
-        return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}'
+        return f"InactiveStreamClosed: Connection was closed without sending the request {self.request!r}"
 
 
 class InvalidHostname(H2Error):
-
-    def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
+    def __init__(
+        self, request: Request, expected_hostname: str, expected_netloc: str
+    ) -> None:
         self.request = request
         self.expected_hostname = expected_hostname
         self.expected_netloc = expected_netloc
 
     def __str__(self) -> str:
-        return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
+        return f"InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}"
 
 
 class StreamCloseReason(Enum):
@@ -100,28 +101,31 @@ class Stream:
         self._request: Request = request
         self._protocol: "H2ClientProtocol" = protocol
 
-        self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
-        self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
+        self._download_maxsize = self._request.meta.get(
+            "download_maxsize", download_maxsize
+        )
+        self._download_warnsize = self._request.meta.get(
+            "download_warnsize", download_warnsize
+        )
 
         # Metadata of an HTTP/2 connection stream
         # initialized when stream is instantiated
         self.metadata: Dict = {
-            'request_content_length': 0 if self._request.body is None else len(self._request.body),
-
+            "request_content_length": 0
+            if self._request.body is None
+            else len(self._request.body),
             # Flag to keep track whether the stream has initiated the request
-            'request_sent': False,
-
+            "request_sent": False,
             # Flag to track whether we have logged about exceeding download warnsize
-            'reached_warnsize': False,
-
+            "reached_warnsize": False,
             # Each time we send a data frame, we will decrease value by the amount send.
-            'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
-
+            "remaining_content_length": 0
+            if self._request.body is None
+            else len(self._request.body),
             # Flag to keep track whether client (self) have closed this stream
-            'stream_closed_local': False,
-
+            "stream_closed_local": False,
             # Flag to keep track whether the server has closed the stream
-            'stream_closed_server': False,
+            "stream_closed_server": False,
         }
 
         # Private variable used to build the response
@@ -130,21 +134,19 @@ class Stream:
         self._response: Dict = {
             # Data received frame by frame from the server is appended
             # and passed to the response Deferred when completely received.
-            'body': BytesIO(),
-
+            "body": BytesIO(),
             # The amount of data received that counts against the
             # flow control window
-            'flow_controlled_size': 0,
-
+            "flow_controlled_size": 0,
             # Headers received after sending the request
-            'headers': Headers({}),
+            "headers": Headers({}),
         }
 
         def _cancel(_) -> None:
             # Close this stream as gracefully as possible
             # If the associated request is initiated we reset this stream
             # else we directly call close() method
-            if self.metadata['request_sent']:
+            if self.metadata["request_sent"]:
                 self.reset_stream(StreamCloseReason.CANCELLED)
             else:
                 self.close(StreamCloseReason.CANCELLED)
@@ -152,7 +154,7 @@ class Stream:
         self._deferred_response = Deferred(_cancel)
 
     def __repr__(self) -> str:
-        return f'Stream(id={self.stream_id!r})'
+        return f"Stream(id={self.stream_id!r})"
 
     @property
     def _log_warnsize(self) -> bool:
@@ -163,14 +165,16 @@ class Stream:
             True if both the above conditions hold true
             False if any of the conditions is false
         """
-        content_length_header = int(self._response['headers'].get(b'Content-Length', -1))
+        content_length_header = int(
+            self._response["headers"].get(b"Content-Length", -1)
+        )
         return (
             self._download_warnsize
             and (
-                self._response['flow_controlled_size'] > self._download_warnsize
+                self._response["flow_controlled_size"] > self._download_warnsize
                 or content_length_header > self._download_warnsize
             )
-            and not self.metadata['reached_warnsize']
+            and not self.metadata["reached_warnsize"]
         )
 
     def get_response(self) -> Deferred:
@@ -183,9 +187,10 @@ class Stream:
         # Make sure that we are sending the request to the correct URL
         url = urlparse(self._request.url)
         return (
-            url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8')
-            or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8')
-            or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
+            url.netloc == str(self._protocol.metadata["uri"].host, "utf-8")
+            or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8")
+            or url.netloc
+            == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
         )
 
     def _get_request_headers(self) -> List[Tuple[str, str]]:
@@ -193,7 +198,7 @@ class Stream:
 
         path = url.path
         if url.query:
-            path += '?' + url.query
+            path += "?" + url.query
 
         # This pseudo-header field MUST NOT be empty for "http" or "https"
         # URIs; "http" or "https" URIs that do not contain a path component
@@ -202,40 +207,40 @@ class Stream:
         # a path component; these MUST include a ":path" pseudo-header field
         # with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
         if not path:
-            path = '*' if self._request.method == 'OPTIONS' else '/'
+            path = "*" if self._request.method == "OPTIONS" else "/"
 
         # Make sure pseudo-headers comes before all the other headers
         headers = [
-            (':method', self._request.method),
-            (':authority', url.netloc),
+            (":method", self._request.method),
+            (":authority", url.netloc),
         ]
 
         # The ":scheme" and ":path" pseudo-header fields MUST
         # be omitted for CONNECT method (refer RFC 7540 - Section 8.3)
-        if self._request.method != 'CONNECT':
+        if self._request.method != "CONNECT":
             headers += [
-                (':scheme', self._protocol.metadata['uri'].scheme),
-                (':path', path),
+                (":scheme", self._protocol.metadata["uri"].scheme),
+                (":path", path),
             ]
 
         content_length = str(len(self._request.body))
-        headers.append(('Content-Length', content_length))
+        headers.append(("Content-Length", content_length))
 
-        content_length_name = self._request.headers.normkey(b'Content-Length')
+        content_length_name = self._request.headers.normkey(b"Content-Length")
         for name, values in self._request.headers.items():
             for value in values:
-                value = str(value, 'utf-8')
+                value = str(value, "utf-8")
                 if name == content_length_name:
                     if value != content_length:
                         logger.warning(
-                            'Ignoring bad Content-Length header %r of request %r, '
-                            'sending %r instead',
+                            "Ignoring bad Content-Length header %r of request %r, "
+                            "sending %r instead",
                             value,
                             self._request,
                             content_length,
                         )
                     continue
-                headers.append((str(name, 'utf-8'), value))
+                headers.append((str(name, "utf-8"), value))
 
         return headers
 
@@ -243,7 +248,7 @@ class Stream:
         if self.check_request_url():
             headers = self._get_request_headers()
             self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
-            self.metadata['request_sent'] = True
+            self.metadata["request_sent"] = True
             self.send_data()
         else:
             # Close this stream calling the response errback
@@ -252,44 +257,53 @@ class Stream:
 
     def send_data(self) -> None:
         """Called immediately after the headers are sent. Here we send all the
-         data as part of the request.
+        data as part of the request.
 
-         If the content length is 0 initially then we end the stream immediately and
-         wait for response data.
+        If the content length is 0 initially then we end the stream immediately and
+        wait for response data.
 
-         Warning: Only call this method when stream not closed from client side
-            and has initiated request already by sending HEADER frame. If not then
-            stream will raise ProtocolError (raise by h2 state machine).
-         """
-        if self.metadata['stream_closed_local']:
+        Warning: Only call this method when stream not closed from client side
+           and has initiated request already by sending HEADER frame. If not then
+           stream will raise ProtocolError (raise by h2 state machine).
+        """
+        if self.metadata["stream_closed_local"]:
             raise StreamClosedError(self.stream_id)
 
         # Firstly, check what the flow control window is for current stream.
-        window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id)
+        window_size = self._protocol.conn.local_flow_control_window(
+            stream_id=self.stream_id
+        )
 
         # Next, check what the maximum frame size is.
         max_frame_size = self._protocol.conn.max_outbound_frame_size
 
         # We will send no more than the window size or the remaining file size
         # of data in this call, whichever is smaller.
-        bytes_to_send_size = min(window_size, self.metadata['remaining_content_length'])
+        bytes_to_send_size = min(window_size, self.metadata["remaining_content_length"])
 
         # We now need to send a number of data frames.
         while bytes_to_send_size > 0:
             chunk_size = min(bytes_to_send_size, max_frame_size)
 
-            data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length']
-            data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
+            data_chunk_start_id = (
+                self.metadata["request_content_length"]
+                - self.metadata["remaining_content_length"]
+            )
+            data_chunk = self._request.body[
+                data_chunk_start_id : data_chunk_start_id + chunk_size
+            ]
 
             self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
             bytes_to_send_size -= chunk_size
-            self.metadata['remaining_content_length'] -= chunk_size
+            self.metadata["remaining_content_length"] -= chunk_size
 
-        self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
+        self.metadata["remaining_content_length"] = max(
+            0, self.metadata["remaining_content_length"]
+        )
 
         # End the stream if no more data needs to be send
-        if self.metadata['remaining_content_length'] == 0:
+        if self.metadata["remaining_content_length"] == 0:
             self._protocol.conn.end_stream(self.stream_id)
 
         # Q. What about the rest of the data?
@@ -301,62 +315,64 @@ class Stream:
         blocked behind the flow control.
         """
         if (
-            self.metadata['remaining_content_length']
-            and not self.metadata['stream_closed_server']
-            and self.metadata['request_sent']
+            self.metadata["remaining_content_length"]
+            and not self.metadata["stream_closed_server"]
+            and self.metadata["request_sent"]
         ):
             self.send_data()
 
     def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
-        self._response['body'].write(data)
-        self._response['flow_controlled_size'] += flow_controlled_length
+        self._response["body"].write(data)
+        self._response["flow_controlled_size"] += flow_controlled_length
 
         # We check maxsize here in case the Content-Length header was not received
-        if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
+        if (
+            self._download_maxsize
+            and self._response["flow_controlled_size"] > self._download_maxsize
+        ):
             self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
             return
 
         if self._log_warnsize:
-            self.metadata['reached_warnsize'] = True
+            self.metadata["reached_warnsize"] = True
             warning_msg = (
                 f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
-                f'warn size ({self._download_warnsize}) in request {self._request}'
+                f"warn size ({self._download_warnsize}) in request {self._request}"
             )
             logger.warning(warning_msg)
 
         # Acknowledge the data received
         self._protocol.conn.acknowledge_received_data(
-            self._response['flow_controlled_size'],
-            self.stream_id
+            self._response["flow_controlled_size"], self.stream_id
         )
 
     def receive_headers(self, headers: List[HeaderTuple]) -> None:
         for name, value in headers:
-            self._response['headers'][name] = value
+            self._response["headers"][name] = value
 
         # Check if we exceed the allowed max data size which can be received
-        expected_size = int(self._response['headers'].get(b'Content-Length', -1))
+        expected_size = int(self._response["headers"].get(b"Content-Length", -1))
         if self._download_maxsize and expected_size > self._download_maxsize:
             self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
             return
 
         if self._log_warnsize:
-            self.metadata['reached_warnsize'] = True
+            self.metadata["reached_warnsize"] = True
             warning_msg = (
-                f'Expected response size ({expected_size}) larger than '
-                f'download warn size ({self._download_warnsize}) in request {self._request}'
+                f"Expected response size ({expected_size}) larger than "
+                f"download warn size ({self._download_warnsize}) in request {self._request}"
             )
             logger.warning(warning_msg)
 
     def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
         """Close this stream by sending a RST_FRAME to the remote peer"""
-        if self.metadata['stream_closed_local']:
+        if self.metadata["stream_closed_local"]:
             raise StreamClosedError(self.stream_id)
 
         # Clear buffer earlier to avoid keeping data in memory for a long time
-        self._response['body'].truncate(0)
+        self._response["body"].truncate(0)
 
-        self.metadata['stream_closed_local'] = True
+        self.metadata["stream_closed_local"] = True
         self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
         self.close(reason)
 
@@ -366,13 +382,14 @@ class Stream:
         errors: Optional[List[BaseException]] = None,
         from_protocol: bool = False,
     ) -> None:
-        """Based on the reason sent we will handle each case.
-        """
-        if self.metadata['stream_closed_server']:
+        """Based on the reason sent we will handle each case."""
+        if self.metadata["stream_closed_server"]:
             raise StreamClosedError(self.stream_id)
 
         if not isinstance(reason, StreamCloseReason):
-            raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
+            raise TypeError(
+                f"Expected StreamCloseReason, received {reason.__class__.__qualname__}"
+            )
 
         # Have default value of errors as an empty list as
         # some cases can add a list of exceptions
@@ -381,7 +398,7 @@ class Stream:
         if not from_protocol:
             self._protocol.pop_stream(self.stream_id)
 
-        self.metadata['stream_closed_server'] = True
+        self.metadata["stream_closed_server"] = True
 
         # We do not check for Content-Length or Transfer-Encoding in response headers
         # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
@@ -392,13 +409,14 @@ class Stream:
         # receiving DATA_FRAME's when we have received the headers (not
         # having Content-Length)
         if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
-            expected_size = int(self._response['headers'].get(
-                b'Content-Length',
-                self._response['flow_controlled_size'])
+            expected_size = int(
+                self._response["headers"].get(
+                    b"Content-Length", self._response["flow_controlled_size"]
+                )
             )
             error_msg = (
-                f'Cancelling download of {self._request.url}: received response '
-                f'size ({expected_size}) larger than download max size ({self._download_maxsize})'
+                f"Cancelling download of {self._request.url}: received response "
+                f"size ({expected_size}) larger than download max size ({self._download_maxsize})"
             )
             logger.error(error_msg)
             self._deferred_response.errback(CancelledError(error_msg))
@@ -416,16 +434,20 @@ class Stream:
 
             # There maybe no :status in headers, we make
             # HTTP Status Code: 499 - Client Closed Request
-            self._response['headers'][':status'] = '499'
+            self._response["headers"][":status"] = "499"
             self._fire_response_deferred()
 
         elif reason is StreamCloseReason.RESET:
-            self._deferred_response.errback(ResponseFailed([
-                Failure(
-                    f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
-                    ProtocolError
+            self._deferred_response.errback(
+                ResponseFailed(
+                    [
+                        Failure(
+                            f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
+                            ProtocolError,
+                        )
+                    ]
                 )
-            ]))
+            )
 
         elif reason is StreamCloseReason.CONNECTION_LOST:
             self._deferred_response.errback(ResponseFailed(errors))
@@ -436,33 +458,35 @@ class Stream:
 
         else:
             assert reason is StreamCloseReason.INVALID_HOSTNAME
-            self._deferred_response.errback(InvalidHostname(
-                self._request,
-                str(self._protocol.metadata['uri'].host, 'utf-8'),
-                f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
-            ))
+            self._deferred_response.errback(
+                InvalidHostname(
+                    self._request,
+                    str(self._protocol.metadata["uri"].host, "utf-8"),
+                    f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}',
+                )
+            )
 
     def _fire_response_deferred(self) -> None:
         """Builds response from the self._response dict
         and fires the response deferred callback with the
         generated response instance"""
 
-        body = self._response['body'].getvalue()
+        body = self._response["body"].getvalue()
         response_cls = responsetypes.from_args(
-            headers=self._response['headers'],
+            headers=self._response["headers"],
             url=self._request.url,
             body=body,
         )
 
         response = response_cls(
             url=self._request.url,
-            status=int(self._response['headers'][':status']),
-            headers=self._response['headers'],
+            status=int(self._response["headers"][":status"]),
+            headers=self._response["headers"],
             body=body,
             request=self._request,
-            certificate=self._protocol.metadata['certificate'],
-            ip_address=self._protocol.metadata['ip_address'],
-            protocol='h2',
+            certificate=self._protocol.metadata["certificate"],
+            ip_address=self._protocol.metadata["ip_address"],
+            protocol="h2",
         )
 
         self._deferred_response.callback(response)
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 6dd5d51a8..1e6fc69e1 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -20,14 +20,18 @@ class BaseSchedulerMeta(type):
     """
     Metaclass to check scheduler classes against the necessary interface
     """
+
     def __instancecheck__(cls, instance):
         return cls.__subclasscheck__(type(instance))
 
     def __subclasscheck__(cls, subclass):
         return (
-            hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests)
-            and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request)
-            and hasattr(subclass, "next_request") and callable(subclass.next_request)
+            hasattr(subclass, "has_pending_requests")
+            and callable(subclass.has_pending_requests)
+            and hasattr(subclass, "enqueue_request")
+            and callable(subclass.enqueue_request)
+            and hasattr(subclass, "next_request")
+            and callable(subclass.next_request)
         )
 
 
@@ -162,6 +166,7 @@ class Scheduler(BaseScheduler):
     :param crawler: The crawler object corresponding to the current crawl.
     :type crawler: :class:`scrapy.crawler.Crawler`
     """
+
     def __init__(
         self,
         dupefilter,
@@ -187,15 +192,15 @@ class Scheduler(BaseScheduler):
         """
         Factory method, initializes the scheduler with arguments taken from the crawl settings
         """
-        dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS'])
+        dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"])
         return cls(
             dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
             jobdir=job_dir(crawler.settings),
-            dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']),
-            mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']),
-            logunser=crawler.settings.getbool('SCHEDULER_DEBUG'),
+            dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]),
+            mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]),
+            logunser=crawler.settings.getbool("SCHEDULER_DEBUG"),
             stats=crawler.stats,
-            pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']),
+            pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]),
             crawler=crawler,
         )
 
@@ -239,11 +244,11 @@ class Scheduler(BaseScheduler):
             return False
         dqok = self._dqpush(request)
         if dqok:
-            self.stats.inc_value('scheduler/enqueued/disk', spider=self.spider)
+            self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
         else:
             self._mqpush(request)
-            self.stats.inc_value('scheduler/enqueued/memory', spider=self.spider)
-        self.stats.inc_value('scheduler/enqueued', spider=self.spider)
+            self.stats.inc_value("scheduler/enqueued/memory", spider=self.spider)
+        self.stats.inc_value("scheduler/enqueued", spider=self.spider)
         return True
 
     def next_request(self) -> Optional[Request]:
@@ -257,13 +262,13 @@ class Scheduler(BaseScheduler):
         """
         request = self.mqs.pop()
         if request is not None:
-            self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
+            self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
         else:
             request = self._dqpop()
             if request is not None:
-                self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider)
+                self.stats.inc_value("scheduler/dequeued/disk", spider=self.spider)
         if request is not None:
-            self.stats.inc_value('scheduler/dequeued', spider=self.spider)
+            self.stats.inc_value("scheduler/dequeued", spider=self.spider)
         return request
 
     def __len__(self) -> int:
@@ -279,13 +284,19 @@ class Scheduler(BaseScheduler):
             self.dqs.push(request)
         except ValueError as e:  # non serializable request
             if self.logunser:
-                msg = ("Unable to serialize request: %(request)s - reason:"
-                       " %(reason)s - no more unserializable requests will be"
-                       " logged (stats being collected)")
-                logger.warning(msg, {'request': request, 'reason': e},
-                               exc_info=True, extra={'spider': self.spider})
+                msg = (
+                    "Unable to serialize request: %(request)s - reason:"
+                    " %(reason)s - no more unserializable requests will be"
+                    " logged (stats being collected)"
+                )
+                logger.warning(
+                    msg,
+                    {"request": request, "reason": e},
+                    exc_info=True,
+                    extra={"spider": self.spider},
+                )
                 self.logunser = False
-            self.stats.inc_value('scheduler/unserializable', spider=self.spider)
+            self.stats.inc_value("scheduler/unserializable", spider=self.spider)
             return False
         else:
             return True
@@ -299,43 +310,50 @@ class Scheduler(BaseScheduler):
         return None
 
     def _mq(self):
-        """ Create a new priority queue instance, with in-memory storage """
-        return create_instance(self.pqclass,
-                               settings=None,
-                               crawler=self.crawler,
-                               downstream_queue_cls=self.mqclass,
-                               key='')
+        """Create a new priority queue instance, with in-memory storage"""
+        return create_instance(
+            self.pqclass,
+            settings=None,
+            crawler=self.crawler,
+            downstream_queue_cls=self.mqclass,
+            key="",
+        )
 
     def _dq(self):
-        """ Create a new priority queue instance, with disk storage """
+        """Create a new priority queue instance, with disk storage"""
         state = self._read_dqs_state(self.dqdir)
-        q = create_instance(self.pqclass,
-                            settings=None,
-                            crawler=self.crawler,
-                            downstream_queue_cls=self.dqclass,
-                            key=self.dqdir,
-                            startprios=state)
+        q = create_instance(
+            self.pqclass,
+            settings=None,
+            crawler=self.crawler,
+            downstream_queue_cls=self.dqclass,
+            key=self.dqdir,
+            startprios=state,
+        )
         if q:
-            logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
-                        {'queuesize': len(q)}, extra={'spider': self.spider})
+            logger.info(
+                "Resuming crawl (%(queuesize)d requests scheduled)",
+                {"queuesize": len(q)},
+                extra={"spider": self.spider},
+            )
         return q
 
     def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
-        """ Return a folder name to keep disk queue state at """
+        """Return a folder name to keep disk queue state at"""
         if jobdir is not None:
-            dqdir = Path(jobdir, 'requests.queue')
+            dqdir = Path(jobdir, "requests.queue")
             if not dqdir.exists():
                 dqdir.mkdir(parents=True)
             return str(dqdir)
         return None
 
     def _read_dqs_state(self, dqdir: str) -> list:
-        path = Path(dqdir, 'active.json')
+        path = Path(dqdir, "active.json")
         if not path.exists():
             return []
         with path.open(encoding="utf-8") as f:
             return json.load(f)
 
     def _write_dqs_state(self, dqdir: str, state: list) -> None:
-        with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f:
+        with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
             json.dump(state, f)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 69ac1cdaf..7c2eefbe6 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -63,7 +63,9 @@ class Slot:
         self.itemproc_size: int = 0
         self.closing: Optional[Deferred] = None
 
-    def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:
+    def add_response_request(
+        self, result: Union[Response, Failure], request: Request
+    ) -> Deferred:
         deferred = Deferred()
         self.queue.append((result, request, deferred))
         if isinstance(result, Response):
@@ -77,7 +79,9 @@ class Slot:
         self.active.add(request)
         return response, request, deferred
 
-    def finish_response(self, result: Union[Response, Failure], request: Request) -> None:
+    def finish_response(
+        self, result: Union[Response, Failure], request: Request
+    ) -> None:
         self.active.remove(request)
         if isinstance(result, Response):
             self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)
@@ -92,13 +96,12 @@ class Slot:
 
 
 class Scraper:
-
     def __init__(self, crawler: Crawler) -> None:
         self.slot: Optional[Slot] = None
         self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
-        itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
+        itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"])
         self.itemproc = itemproc_cls.from_crawler(crawler)
-        self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS')
+        self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS")
         self.crawler = crawler
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
@@ -106,7 +109,7 @@ class Scraper:
     @inlineCallbacks
     def open_spider(self, spider: Spider):
         """Open the given spider for scraping and allocate resources for it"""
-        self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
+        self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
         yield self.itemproc.open_spider(spider)
 
     def close_spider(self, spider: Spider) -> Deferred:
@@ -127,7 +130,9 @@ class Scraper:
         if self.slot.closing and self.slot.is_idle():
             self.slot.closing.callback(spider)
 
-    def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+    def enqueue_scrape(
+        self, result: Union[Response, Failure], request: Request, spider: Spider
+    ) -> Deferred:
         if self.slot is None:
             raise RuntimeError("Scraper slot not assigned")
         dfd = self.slot.add_response_request(result, request)
@@ -140,10 +145,13 @@ class Scraper:
 
         dfd.addBoth(finish_scraping)
         dfd.addErrback(
-            lambda f: logger.error('Scraper bug processing %(request)s',
-                                   {'request': request},
-                                   exc_info=failure_to_exc_info(f),
-                                   extra={'spider': spider}))
+            lambda f: logger.error(
+                "Scraper bug processing %(request)s",
+                {"request": request},
+                exc_info=failure_to_exc_info(f),
+                extra={"spider": spider},
+            )
+        )
         self._scrape_next(spider)
         return dfd
 
@@ -153,35 +161,49 @@ class Scraper:
             response, request, deferred = self.slot.next_response_request_deferred()
             self._scrape(response, request, spider).chainDeferred(deferred)
 
-    def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+    def _scrape(
+        self, result: Union[Response, Failure], request: Request, spider: Spider
+    ) -> Deferred:
         """
         Handle the downloaded response or failure through the spider callback/errback
         """
         if not isinstance(result, (Response, Failure)):
-            raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")
-        dfd = self._scrape2(result, request, spider)  # returns spider's processed output
+            raise TypeError(
+                f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}"
+            )
+        dfd = self._scrape2(
+            result, request, spider
+        )  # returns spider's processed output
         dfd.addErrback(self.handle_spider_error, request, result, spider)
         dfd.addCallback(self.handle_spider_output, request, result, spider)
         return dfd
 
-    def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+    def _scrape2(
+        self, result: Union[Response, Failure], request: Request, spider: Spider
+    ) -> Deferred:
         """
         Handle the different cases of request's result been a Response or a Failure
         """
         if isinstance(result, Response):
-            return self.spidermw.scrape_response(self.call_spider, result, request, spider)
+            return self.spidermw.scrape_response(
+                self.call_spider, result, request, spider
+            )
         # else result is a Failure
         dfd = self.call_spider(result, request, spider)
         return dfd.addErrback(self._log_download_errors, result, request, spider)
 
-    def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+    def call_spider(
+        self, result: Union[Response, Failure], request: Request, spider: Spider
+    ) -> Deferred:
         if isinstance(result, Response):
             if getattr(result, "request", None) is None:
                 result.request = request
             callback = result.request.callback or spider._parse
             warn_on_generator_with_return_value(spider, callback)
             dfd = defer_succeed(result)
-            dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
+            dfd.addCallbacks(
+                callback=callback, callbackKeywords=result.request.cb_kwargs
+            )
         else:  # result is a Failure
             result.request = request
             warn_on_generator_with_return_value(spider, request.errback)
@@ -189,45 +211,69 @@ class Scraper:
             dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
-    def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
+    def handle_spider_error(
+        self, _failure: Failure, request: Request, response: Response, spider: Spider
+    ) -> None:
         exc = _failure.value
         if isinstance(exc, CloseSpider):
             assert self.crawler.engine is not None  # typing
-            self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
+            self.crawler.engine.close_spider(spider, exc.reason or "cancelled")
             return
         logkws = self.logformatter.spider_error(_failure, request, response, spider)
         logger.log(
             *logformatter_adapter(logkws),
             exc_info=failure_to_exc_info(_failure),
-            extra={'spider': spider}
+            extra={"spider": spider},
         )
         self.signals.send_catch_log(
             signal=signals.spider_error,
-            failure=_failure, response=response,
-            spider=spider
+            failure=_failure,
+            response=response,
+            spider=spider,
         )
         self.crawler.stats.inc_value(
-            f"spider_exceptions/{_failure.value.__class__.__name__}",
-            spider=spider
+            f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider
         )
 
-    def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
-                             response: Response, spider: Spider) -> Deferred:
+    def handle_spider_output(
+        self,
+        result: Union[Iterable, AsyncIterable],
+        request: Request,
+        response: Response,
+        spider: Spider,
+    ) -> Deferred:
         if not result:
             return defer_succeed(None)
         it: Union[Generator, AsyncGenerator]
         if isinstance(result, AsyncIterable):
-            it = aiter_errback(result, self.handle_spider_error, request, response, spider)
-            dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
-                                 request, response, spider)
+            it = aiter_errback(
+                result, self.handle_spider_error, request, response, spider
+            )
+            dfd = parallel_async(
+                it,
+                self.concurrent_items,
+                self._process_spidermw_output,
+                request,
+                response,
+                spider,
+            )
         else:
-            it = iter_errback(result, self.handle_spider_error, request, response, spider)
-            dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
-                           request, response, spider)
+            it = iter_errback(
+                result, self.handle_spider_error, request, response, spider
+            )
+            dfd = parallel(
+                it,
+                self.concurrent_items,
+                self._process_spidermw_output,
+                request,
+                response,
+                spider,
+            )
         return dfd
 
-    def _process_spidermw_output(self, output: Any, request: Request, response: Response,
-                                 spider: Spider) -> Optional[Deferred]:
+    def _process_spidermw_output(
+        self, output: Any, request: Request, response: Response, spider: Spider
+    ) -> Optional[Deferred]:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider
         """
@@ -245,14 +291,19 @@ class Scraper:
         else:
             typename = type(output).__name__
             logger.error(
-                'Spider must return request, item, or None, got %(typename)r in %(request)s',
-                {'request': request, 'typename': typename},
-                extra={'spider': spider},
+                "Spider must return request, item, or None, got %(typename)r in %(request)s",
+                {"request": request, "typename": typename},
+                extra={"spider": spider},
             )
         return None
 
-    def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request,
-                             spider: Spider) -> Union[Failure, None]:
+    def _log_download_errors(
+        self,
+        spider_failure: Failure,
+        download_failure: Failure,
+        request: Request,
+        spider: Spider,
+    ) -> Union[Failure, None]:
         """Log and silence errors that come from the engine (typically download
         errors that got propagated thru here).
 
@@ -262,29 +313,33 @@ class Scraper:
         """
         if not download_failure.check(IgnoreRequest):
             if download_failure.frames:
-                logkws = self.logformatter.download_error(download_failure, request, spider)
+                logkws = self.logformatter.download_error(
+                    download_failure, request, spider
+                )
                 logger.log(
                     *logformatter_adapter(logkws),
-                    extra={'spider': spider},
+                    extra={"spider": spider},
                     exc_info=failure_to_exc_info(download_failure),
                 )
             else:
                 errmsg = download_failure.getErrorMessage()
                 if errmsg:
                     logkws = self.logformatter.download_error(
-                        download_failure, request, spider, errmsg)
+                        download_failure, request, spider, errmsg
+                    )
                     logger.log(
                         *logformatter_adapter(logkws),
-                        extra={'spider': spider},
+                        extra={"spider": spider},
                     )
 
         if spider_failure is not download_failure:
             return spider_failure
         return None
 
-    def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None:
-        """ItemProcessor finished for the given ``item`` and returned ``output``
-        """
+    def _itemproc_finished(
+        self, output: Any, item: Any, response: Response, spider: Spider
+    ) -> None:
+        """ItemProcessor finished for the given ``item`` and returned ``output``"""
         assert self.slot is not None  # typing
         self.slot.itemproc_size -= 1
         if isinstance(output, Failure):
@@ -292,19 +347,30 @@ class Scraper:
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
                 if logkws is not None:
-                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                    logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
                 return self.signals.send_catch_log_deferred(
-                    signal=signals.item_dropped, item=item, response=response,
-                    spider=spider, exception=output.value)
+                    signal=signals.item_dropped,
+                    item=item,
+                    response=response,
+                    spider=spider,
+                    exception=output.value,
+                )
             logkws = self.logformatter.item_error(item, ex, response, spider)
-            logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
-                       exc_info=failure_to_exc_info(output))
+            logger.log(
+                *logformatter_adapter(logkws),
+                extra={"spider": spider},
+                exc_info=failure_to_exc_info(output),
+            )
             return self.signals.send_catch_log_deferred(
-                signal=signals.item_error, item=item, response=response,
-                spider=spider, failure=output)
+                signal=signals.item_error,
+                item=item,
+                response=response,
+                spider=spider,
+                failure=output,
+            )
         logkws = self.logformatter.scraped(output, response, spider)
         if logkws is not None:
-            logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+            logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
         return self.signals.send_catch_log_deferred(
-            signal=signals.item_scraped, item=output, response=response,
-            spider=spider)
+            signal=signals.item_scraped, item=output, response=response, spider=spider
+        )
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 91e4b9cb4..1aaed5865 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -6,7 +6,17 @@ See documentation in docs/topics/spider-middleware.rst
 import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
-from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
+from typing import (
+    Any,
+    AsyncGenerator,
+    AsyncIterable,
+    Callable,
+    Generator,
+    Iterable,
+    Tuple,
+    Union,
+    cast,
+)
 
 from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
@@ -17,7 +27,12 @@ from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import (
+    mustbe_deferred,
+    deferred_from_coro,
+    deferred_f_from_coro_f,
+    maybe_deferred_to_future,
+)
 from scrapy.utils.python import MutableAsyncChain, MutableChain
 
 
@@ -33,7 +48,7 @@ def _isiterable(o) -> bool:
 
 class SpiderMiddlewareManager(MiddlewareManager):
 
-    component_name = 'spider middleware'
+    component_name = "spider middleware"
 
     def __init__(self, *middlewares):
         super().__init__(*middlewares)
@@ -41,28 +56,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
+        return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
 
     def _add_middleware(self, mw):
         super()._add_middleware(mw)
-        if hasattr(mw, 'process_spider_input'):
-            self.methods['process_spider_input'].append(mw.process_spider_input)
-        if hasattr(mw, 'process_start_requests'):
-            self.methods['process_start_requests'].appendleft(mw.process_start_requests)
-        process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
-        self.methods['process_spider_output'].appendleft(process_spider_output)
-        process_spider_exception = getattr(mw, 'process_spider_exception', None)
-        self.methods['process_spider_exception'].appendleft(process_spider_exception)
+        if hasattr(mw, "process_spider_input"):
+            self.methods["process_spider_input"].append(mw.process_spider_input)
+        if hasattr(mw, "process_start_requests"):
+            self.methods["process_start_requests"].appendleft(mw.process_start_requests)
+        process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
+        self.methods["process_spider_output"].appendleft(process_spider_output)
+        process_spider_exception = getattr(mw, "process_spider_exception", None)
+        self.methods["process_spider_exception"].appendleft(process_spider_exception)
 
-    def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
-                              spider: Spider) -> Any:
-        for method in self.methods['process_spider_input']:
+    def _process_spider_input(
+        self,
+        scrape_func: ScrapeFunc,
+        response: Response,
+        request: Request,
+        spider: Spider,
+    ) -> Any:
+        for method in self.methods["process_spider_input"]:
             method = cast(Callable, method)
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"{method.__qualname__} must return None "
-                           f"or raise an exception, got {type(result)}")
+                    msg = (
+                        f"{method.__qualname__} must return None "
+                        f"or raise an exception, got {type(result)}"
+                    )
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
                 raise
@@ -70,17 +92,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 return scrape_func(Failure(), request, spider)
         return scrape_func(response, request, spider)
 
-    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
-                           exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
-                           ) -> Union[Generator, AsyncGenerator]:
-
+    def _evaluate_iterable(
+        self,
+        response: Response,
+        spider: Spider,
+        iterable: Union[Iterable, AsyncIterable],
+        exception_processor_index: int,
+        recover_to: Union[MutableChain, MutableAsyncChain],
+    ) -> Union[Generator, AsyncGenerator]:
         def process_sync(iterable: Iterable):
             try:
                 for r in iterable:
                     yield r
             except Exception as ex:
-                exception_result = self._process_spider_exception(response, spider, Failure(ex),
-                                                                  exception_processor_index)
+                exception_result = self._process_spider_exception(
+                    response, spider, Failure(ex), exception_processor_index
+                )
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)
@@ -90,8 +117,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 async for r in iterable:
                     yield r
             except Exception as ex:
-                exception_result = self._process_spider_exception(response, spider, Failure(ex),
-                                                                  exception_processor_index)
+                exception_result = self._process_spider_exception(
+                    response, spider, Failure(ex), exception_processor_index
+                )
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)
@@ -100,13 +128,20 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return process_async(iterable)
         return process_sync(iterable)
 
-    def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
-                                  start_index: int = 0) -> Union[Failure, MutableChain]:
+    def _process_spider_exception(
+        self,
+        response: Response,
+        spider: Spider,
+        _failure: Failure,
+        start_index: int = 0,
+    ) -> Union[Failure, MutableChain]:
         exception = _failure.value
         # don't handle _InvalidOutput exception
         if isinstance(exception, _InvalidOutput):
             return _failure
-        method_list = islice(self.methods['process_spider_exception'], start_index, None)
+        method_list = islice(
+            self.methods["process_spider_exception"], start_index, None
+        )
         for method_index, method in enumerate(method_list, start=start_index):
             if method is None:
                 continue
@@ -115,7 +150,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
+                dfd: Deferred = self._process_spider_output(
+                    response, spider, result, method_index + 1
+                )
                 # _process_spider_output() returns a Deferred only because of downgrading so this can be
                 # simplified when downgrading is removed.
                 if dfd.called:
@@ -128,8 +165,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
             elif result is None:
                 continue
             else:
-                msg = (f"{method.__qualname__} must return None "
-                       f"or an iterable, got {type(result)}")
+                msg = (
+                    f"{method.__qualname__} must return None "
+                    f"or an iterable, got {type(result)}"
+                )
                 raise _InvalidOutput(msg)
         return _failure
 
@@ -137,9 +176,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
     # being available immediately which doesn't work when it's a wrapped coroutine.
     # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
     @inlineCallbacks
-    def _process_spider_output(self, response: Response, spider: Spider,
-                               result: Union[Iterable, AsyncIterable], start_index: int = 0
-                               ) -> Deferred:
+    def _process_spider_output(
+        self,
+        response: Response,
+        spider: Spider,
+        result: Union[Iterable, AsyncIterable],
+        start_index: int = 0,
+    ) -> Deferred:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered: Union[MutableChain, MutableAsyncChain]
@@ -156,7 +199,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         # Storing methods and method tuples in the same list is weird but we should be able to roll this back
         # when we drop this compatibility feature.
 
-        method_list = islice(self.methods['process_spider_output'], start_index, None)
+        method_list = islice(self.methods["process_spider_output"], start_index, None)
         for method_index, method_pair in enumerate(method_list, start=start_index):
             if method_pair is None:
                 continue
@@ -177,24 +220,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     result = as_async_generator(result)
                 elif need_downgrade:
                     if not self.downgrade_warning_done:
-                        logger.warning(f"Async iterable passed to {method.__qualname__} "
-                                       f"was downgraded to a non-async one")
+                        logger.warning(
+                            f"Async iterable passed to {method.__qualname__} "
+                            f"was downgraded to a non-async one"
+                        )
                         self.downgrade_warning_done = True
                     assert isinstance(result, AsyncIterable)
                     # AsyncIterable -> Iterable
                     result = yield deferred_from_coro(collect_asyncgen(result))
                     if isinstance(recovered, AsyncIterable):
-                        recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
+                        recovered_collected = yield deferred_from_coro(
+                            collect_asyncgen(recovered)
+                        )
                         recovered = MutableChain(recovered_collected)
                 # might fail directly if the output value is not a generator
                 result = method(response=response, result=result, spider=spider)
             except Exception as ex:
-                exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1)
+                exception_result = self._process_spider_exception(
+                    response, spider, Failure(ex), method_index + 1
+                )
                 if isinstance(exception_result, Failure):
                     raise
                 return exception_result
             if _isiterable(result):
-                result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
+                result = self._evaluate_iterable(
+                    response, spider, result, method_index + 1, recovered
+                )
             else:
                 if iscoroutine(result):
                     result.close()  # Silence warning about not awaiting
@@ -214,15 +265,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return MutableAsyncChain(result, recovered)
         return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
-                                       ) -> Union[MutableChain, MutableAsyncChain]:
+    async def _process_callback_output(
+        self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+    ) -> Union[MutableChain, MutableAsyncChain]:
         recovered: Union[MutableChain, MutableAsyncChain]
         if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
+        result = await maybe_deferred_to_future(
+            self._process_spider_output(response, spider, result)
+        )
         if isinstance(result, AsyncIterable):
             return MutableAsyncChain(result, recovered)
         if isinstance(recovered, AsyncIterable):
@@ -230,41 +284,60 @@ class SpiderMiddlewareManager(MiddlewareManager):
             recovered = MutableChain(recovered_collected)
         return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
-                        spider: Spider) -> Deferred:
-        async def process_callback_output(result: Union[Iterable, AsyncIterable]
-                                          ) -> Union[MutableChain, MutableAsyncChain]:
+    def scrape_response(
+        self,
+        scrape_func: ScrapeFunc,
+        response: Response,
+        request: Request,
+        spider: Spider,
+    ) -> Deferred:
+        async def process_callback_output(
+            result: Union[Iterable, AsyncIterable]
+        ) -> Union[MutableChain, MutableAsyncChain]:
             return await self._process_callback_output(response, spider, result)
 
         def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
             return self._process_spider_exception(response, spider, _failure)
 
-        dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
-        dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
+        dfd = mustbe_deferred(
+            self._process_spider_input, scrape_func, response, request, spider
+        )
+        dfd.addCallbacks(
+            callback=deferred_f_from_coro_f(process_callback_output),
+            errback=process_spider_exception,
+        )
         return dfd
 
     def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
-        return self._process_chain('process_start_requests', start_requests, spider)
+        return self._process_chain("process_start_requests", start_requests, spider)
 
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
-    def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
+    def _get_async_method_pair(
+        mw: Any, methodname: str
+    ) -> Union[None, Callable, Tuple[Callable, Callable]]:
         normal_method = getattr(mw, methodname, None)
         methodname_async = methodname + "_async"
         async_method = getattr(mw, methodname_async, None)
         if not async_method:
             return normal_method
         if not normal_method:
-            logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
-                         f"without {methodname}, skipping this method.")
+            logger.error(
+                f"Middleware {mw.__qualname__} has {methodname_async} "
+                f"without {methodname}, skipping this method."
+            )
             return None
         if not isasyncgenfunction(async_method):
-            logger.error(f"{async_method.__qualname__} is not "
-                         f"an async generator function, skipping this method.")
+            logger.error(
+                f"{async_method.__qualname__} is not "
+                f"an async generator function, skipping this method."
+            )
             return normal_method
         if isasyncgenfunction(normal_method):
-            logger.error(f"{normal_method.__qualname__} is an async "
-                         f"generator function while {methodname_async} exists, "
-                         f"skipping both methods.")
+            logger.error(
+                f"{normal_method.__qualname__} is an async "
+                f"generator function while {methodname_async} exists, "
+                f"skipping both methods."
+            )
             return None
         return normal_method, async_method
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4700a30ab..f58cd73d3 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -49,10 +49,9 @@ logger = logging.getLogger(__name__)
 
 
 class Crawler:
-
     def __init__(self, spidercls, settings=None, init_reactor: bool = False):
         if isinstance(spidercls, Spider):
-            raise ValueError('The spidercls argument must be a class, not an object')
+            raise ValueError("The spidercls argument must be a class, not an object")
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
@@ -63,14 +62,15 @@ class Crawler:
 
         self.signals = SignalManager(self)
 
-        self.stats = load_object(self.settings['STATS_CLASS'])(self)
+        self.stats = load_object(self.settings["STATS_CLASS"])(self)
 
-        handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
+        handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
         logging.root.addHandler(handler)
 
         d = dict(overridden_settings(self.settings))
-        logger.info("Overridden settings:\n%(settings)s",
-                    {'settings': pprint.pformat(d)})
+        logger.info(
+            "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
+        )
 
         if get_scrapy_root_handler() is not None:
             # scrapy root handler already installed: update it with new settings
@@ -80,11 +80,11 @@ class Crawler:
         self.__remove_handler = lambda: logging.root.removeHandler(handler)
         self.signals.connect(self.__remove_handler, signals.engine_stopped)
 
-        lf_cls = load_object(self.settings['LOG_FORMATTER'])
+        lf_cls = load_object(self.settings["LOG_FORMATTER"])
         self.logformatter = lf_cls.from_crawler(self)
 
         self.request_fingerprinter: RequestFingerprinter = create_instance(
-            load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+            load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
             settings=self.settings,
             crawler=self,
         )
@@ -160,23 +160,26 @@ class CrawlerRunner:
     crawlers = property(
         lambda self: self._crawlers,
         doc="Set of :class:`crawlers <scrapy.crawler.Crawler>` started by "
-            ":meth:`crawl` and managed by this class."
+        ":meth:`crawl` and managed by this class.",
     )
 
     @staticmethod
     def _get_spider_loader(settings):
-        """ Get SpiderLoader instance from settings """
-        cls_path = settings.get('SPIDER_LOADER_CLASS')
+        """Get SpiderLoader instance from settings"""
+        cls_path = settings.get("SPIDER_LOADER_CLASS")
         loader_cls = load_object(cls_path)
-        excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
+        excs = (
+            (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
+        )
         try:
             verifyClass(ISpiderLoader, loader_cls)
         except excs:
             warnings.warn(
-                'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
-                'not fully implement scrapy.interfaces.ISpiderLoader interface. '
-                'Please add all missing methods to avoid unexpected runtime errors.',
-                category=ScrapyDeprecationWarning, stacklevel=2
+                "SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does "
+                "not fully implement scrapy.interfaces.ISpiderLoader interface. "
+                "Please add all missing methods to avoid unexpected runtime errors.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
             )
         return loader_cls.from_settings(settings.frozencopy())
 
@@ -191,9 +194,12 @@ class CrawlerRunner:
 
     @property
     def spiders(self):
-        warnings.warn("CrawlerRunner.spiders attribute is renamed to "
-                      "CrawlerRunner.spider_loader.",
-                      category=ScrapyDeprecationWarning, stacklevel=2)
+        warnings.warn(
+            "CrawlerRunner.spiders attribute is renamed to "
+            "CrawlerRunner.spider_loader.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         return self.spider_loader
 
     def crawl(self, crawler_or_spidercls, *args, **kwargs):
@@ -220,8 +226,9 @@ class CrawlerRunner:
         """
         if isinstance(crawler_or_spidercls, Spider):
             raise ValueError(
-                'The crawler_or_spidercls argument cannot be a spider object, '
-                'it must be a spider class (or a Crawler object)')
+                "The crawler_or_spidercls argument cannot be a spider object, "
+                "it must be a spider class (or a Crawler object)"
+            )
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
@@ -233,7 +240,7 @@ class CrawlerRunner:
         def _done(result):
             self.crawlers.discard(crawler)
             self._active.discard(d)
-            self.bootstrap_failed |= not getattr(crawler, 'spider', None)
+            self.bootstrap_failed |= not getattr(crawler, "spider", None)
             return result
 
         return d.addBoth(_done)
@@ -251,8 +258,9 @@ class CrawlerRunner:
         """
         if isinstance(crawler_or_spidercls, Spider):
             raise ValueError(
-                'The crawler_or_spidercls argument cannot be a spider object, '
-                'it must be a spider class (or a Crawler object)')
+                "The crawler_or_spidercls argument cannot be a spider object, "
+                "it must be a spider class (or a Crawler object)"
+            )
         if isinstance(crawler_or_spidercls, Crawler):
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)
@@ -314,18 +322,23 @@ class CrawlerProcess(CrawlerRunner):
 
     def _signal_shutdown(self, signum, _):
         from twisted.internet import reactor
+
         install_shutdown_handlers(self._signal_kill)
         signame = signal_names[signum]
-        logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
-                    {'signame': signame})
+        logger.info(
+            "Received %(signame)s, shutting down gracefully. Send again to force ",
+            {"signame": signame},
+        )
         reactor.callFromThread(self._graceful_stop_reactor)
 
     def _signal_kill(self, signum, _):
         from twisted.internet import reactor
+
         install_shutdown_handlers(signal.SIG_IGN)
         signame = signal_names[signum]
-        logger.info('Received %(signame)s twice, forcing unclean shutdown',
-                    {'signame': signame})
+        logger.info(
+            "Received %(signame)s twice, forcing unclean shutdown", {"signame": signame}
+        )
         reactor.callFromThread(self._stop_reactor)
 
     def _create_crawler(self, spidercls):
@@ -351,6 +364,7 @@ class CrawlerProcess(CrawlerRunner):
             handlers (default: True)
         """
         from twisted.internet import reactor
+
         if stop_after_crawl:
             d = self.join()
             # Don't start the reactor if the deferreds are already fired
@@ -364,8 +378,8 @@ class CrawlerProcess(CrawlerRunner):
         resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
         resolver.install_on_reactor()
         tp = reactor.getThreadPool()
-        tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
-        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
+        tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
+        reactor.addSystemEventTrigger("before", "shutdown", self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
     def _graceful_stop_reactor(self):
@@ -375,6 +389,7 @@ class CrawlerProcess(CrawlerRunner):
 
     def _stop_reactor(self, _=None):
         from twisted.internet import reactor
+
         try:
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 4e12a5044..86ff7b9fe 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -17,14 +17,14 @@ class AjaxCrawlMiddleware:
     """
 
     def __init__(self, settings):
-        if not settings.getbool('AJAXCRAWL_ENABLED'):
+        if not settings.getbool("AJAXCRAWL_ENABLED"):
             raise NotConfigured
 
         # XXX: Google parses at least first 100k bytes; scrapy's redirect
         # middleware parses first 4k. 4k turns out to be insufficient
         # for this middleware, and parsing 100k could be slow.
         # We use something in between (32K) by default.
-        self.lookup_bytes = settings.getint('AJAXCRAWL_MAXSIZE', 32768)
+        self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -35,23 +35,25 @@ class AjaxCrawlMiddleware:
         if not isinstance(response, HtmlResponse) or response.status != 200:
             return response
 
-        if request.method != 'GET':
+        if request.method != "GET":
             # other HTTP methods are either not safe or don't have a body
             return response
 
-        if 'ajax_crawlable' in request.meta:  # prevent loops
+        if "ajax_crawlable" in request.meta:  # prevent loops
             return response
 
         if not self._has_ajax_crawlable_variant(response):
             return response
 
         # scrapy already handles #! links properly
-        ajax_crawl_request = request.replace(url=request.url + '#!')
-        logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
-                     {'ajax_crawl_request': ajax_crawl_request, 'request': request},
-                     extra={'spider': spider})
+        ajax_crawl_request = request.replace(url=request.url + "#!")
+        logger.debug(
+            "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
+            {"ajax_crawl_request": ajax_crawl_request, "request": request},
+            extra={"spider": spider},
+        )
 
-        ajax_crawl_request.meta['ajax_crawlable'] = True
+        ajax_crawl_request.meta["ajax_crawlable"] = True
         return ajax_crawl_request
 
     def _has_ajax_crawlable_variant(self, response):
@@ -59,12 +61,14 @@ class AjaxCrawlMiddleware:
         Return True if a page without hash fragment could be "AJAX crawlable"
         according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
         """
-        body = response.text[:self.lookup_bytes]
+        body = response.text[: self.lookup_bytes]
         return _has_ajaxcrawlable_meta(body)
 
 
 # XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
+_ajax_crawlable_re = re.compile(
+    r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'
+)
 
 
 def _has_ajaxcrawlable_meta(text):
@@ -82,12 +86,12 @@ def _has_ajaxcrawlable_meta(text):
     # Stripping scripts and comments is slow (about 20x slower than
     # just checking if a string is in text); this is a quick fail-fast
     # path that should work for most pages.
-    if 'fragment' not in text:
+    if "fragment" not in text:
         return False
-    if 'content' not in text:
+    if "content" not in text:
         return False
 
-    text = html.remove_tags_with_content(text, ('script', 'noscript'))
+    text = html.remove_tags_with_content(text, ("script", "noscript"))
     text = html.replace_entities(text)
     text = html.remove_comments(text)
     return _ajax_crawlable_re.search(text) is not None
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 3cba269f1..6495157d7 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -29,14 +29,14 @@ class CookiesMiddleware:
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('COOKIES_ENABLED'):
+        if not crawler.settings.getbool("COOKIES_ENABLED"):
             raise NotConfigured
-        return cls(crawler.settings.getbool('COOKIES_DEBUG'))
+        return cls(crawler.settings.getbool("COOKIES_DEBUG"))
 
     def _process_cookies(self, cookies, *, jar, request):
         for cookie in cookies:
             cookie_domain = cookie.domain
-            if cookie_domain.startswith('.'):
+            if cookie_domain.startswith("."):
                 cookie_domain = cookie_domain[1:]
 
             request_domain = urlparse_cached(request).hostname.lower()
@@ -49,7 +49,7 @@ class CookiesMiddleware:
             jar.set_cookie_if_ok(cookie, request)
 
     def process_request(self, request, spider):
-        if request.meta.get('dont_merge_cookies', False):
+        if request.meta.get("dont_merge_cookies", False):
             return
 
         cookiejarkey = request.meta.get("cookiejar")
@@ -58,12 +58,12 @@ class CookiesMiddleware:
         self._process_cookies(cookies, jar=jar, request=request)
 
         # set Cookie header
-        request.headers.pop('Cookie', None)
+        request.headers.pop("Cookie", None)
         jar.add_cookie_header(request)
         self._debug_cookie(request, spider)
 
     def process_response(self, request, response, spider):
-        if request.meta.get('dont_merge_cookies', False):
+        if request.meta.get("dont_merge_cookies", False):
             return response
 
         # extract cookies from Set-Cookie and drop invalid/expired cookies
@@ -78,21 +78,25 @@ class CookiesMiddleware:
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = [to_unicode(c, errors='replace')
-                  for c in request.headers.getlist('Cookie')]
+            cl = [
+                to_unicode(c, errors="replace")
+                for c in request.headers.getlist("Cookie")
+            ]
             if cl:
                 cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
                 msg = f"Sending cookies to: {request}\n{cookies}"
-                logger.debug(msg, extra={'spider': spider})
+                logger.debug(msg, extra={"spider": spider})
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = [to_unicode(c, errors='replace')
-                  for c in response.headers.getlist('Set-Cookie')]
+            cl = [
+                to_unicode(c, errors="replace")
+                for c in response.headers.getlist("Set-Cookie")
+            ]
             if cl:
                 cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
                 msg = f"Received cookies from: {response}\n{cookies}"
-                logger.debug(msg, extra={'spider': spider})
+                logger.debug(msg, extra={"spider": spider})
 
     def _format_cookie(self, cookie, request):
         """
@@ -113,8 +117,11 @@ class CookiesMiddleware:
                 try:
                     decoded[key] = cookie[key].decode("utf8")
                 except UnicodeDecodeError:
-                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
-                                   request, cookie)
+                    logger.warning(
+                        "Non UTF-8 encoded cookie found in request %s: %s",
+                        request,
+                        cookie,
+                    )
                     decoded[key] = cookie[key].decode("latin1", errors="replace")
 
         cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index e01e9cc76..410015281 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes
 
 
 warn(
-    'scrapy.downloadermiddlewares.decompression is deprecated',
+    "scrapy.downloadermiddlewares.decompression is deprecated",
     ScrapyDeprecationWarning,
     stacklevel=2,
 )
@@ -26,15 +26,15 @@ logger = logging.getLogger(__name__)
 
 
 class DecompressionMiddleware:
-    """ This middleware tries to recognise and extract the possibly compressed
-    responses that may arrive. """
+    """This middleware tries to recognise and extract the possibly compressed
+    responses that may arrive."""
 
     def __init__(self):
         self._formats = {
-            'tar': self._is_tar,
-            'zip': self._is_zip,
-            'gz': self._is_gzip,
-            'bz2': self._is_bzip2
+            "tar": self._is_tar,
+            "zip": self._is_zip,
+            "gz": self._is_gzip,
+            "bz2": self._is_bzip2,
         }
 
     def _is_tar(self, response):
@@ -86,7 +86,10 @@ class DecompressionMiddleware:
         for fmt, func in self._formats.items():
             new_response = func(response)
             if new_response:
-                logger.debug('Decompressed response with format: %(responsefmt)s',
-                             {'responsefmt': fmt}, extra={'spider': spider})
+                logger.debug(
+                    "Decompressed response with format: %(responsefmt)s",
+                    {"responsefmt": fmt},
+                    extra={"spider": spider},
+                )
                 return new_response
         return response
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index f67961881..cdacc7368 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -8,13 +8,12 @@ from scrapy.utils.python import without_none_values
 
 
 class DefaultHeadersMiddleware:
-
     def __init__(self, headers):
         self._headers = headers
 
     @classmethod
     def from_crawler(cls, crawler):
-        headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
+        headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
         return cls(headers.items())
 
     def process_request(self, request, spider):
diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py
index d373a22df..a926ecf56 100644
--- a/scrapy/downloadermiddlewares/downloadtimeout.py
+++ b/scrapy/downloadermiddlewares/downloadtimeout.py
@@ -8,19 +8,18 @@ from scrapy import signals
 
 
 class DownloadTimeoutMiddleware:
-
     def __init__(self, timeout=180):
         self._timeout = timeout
 
     @classmethod
     def from_crawler(cls, crawler):
-        o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
+        o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         return o
 
     def spider_opened(self, spider):
-        self._timeout = getattr(spider, 'download_timeout', self._timeout)
+        self._timeout = getattr(spider, "download_timeout", self._timeout)
 
     def process_request(self, request, spider):
         if self._timeout:
-            request.meta.setdefault('download_timeout', self._timeout)
+            request.meta.setdefault("download_timeout", self._timeout)
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 1bee3e279..de5a81388 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -24,27 +24,29 @@ class HttpAuthMiddleware:
         return o
 
     def spider_opened(self, spider):
-        usr = getattr(spider, 'http_user', '')
-        pwd = getattr(spider, 'http_pass', '')
+        usr = getattr(spider, "http_user", "")
+        pwd = getattr(spider, "http_pass", "")
         if usr or pwd:
             self.auth = basic_auth_header(usr, pwd)
-            if not hasattr(spider, 'http_auth_domain'):
-                warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
-                              'problems if the spider makes requests to several different domains. http_auth_domain '
-                              'will be set to the domain of the first request, please set it to the correct value '
-                              'explicitly.',
-                              category=ScrapyDeprecationWarning)
+            if not hasattr(spider, "http_auth_domain"):
+                warnings.warn(
+                    "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "
+                    "problems if the spider makes requests to several different domains. http_auth_domain "
+                    "will be set to the domain of the first request, please set it to the correct value "
+                    "explicitly.",
+                    category=ScrapyDeprecationWarning,
+                )
                 self.domain_unset = True
             else:
                 self.domain = spider.http_auth_domain
                 self.domain_unset = False
 
     def process_request(self, request, spider):
-        auth = getattr(self, 'auth', None)
-        if auth and b'Authorization' not in request.headers:
+        auth = getattr(self, "auth", None)
+        if auth and b"Authorization" not in request.headers:
             domain = urlparse_cached(request).hostname
             if self.domain_unset:
                 self.domain = domain
                 self.domain_unset = False
             if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
-                request.headers[b'Authorization'] = auth
+                request.headers[b"Authorization"] = auth
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 80ed7ac75..eb2754f1d 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -29,21 +29,31 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlew
 
 class HttpCacheMiddleware:
 
-    DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
-                           ConnectionRefusedError, ConnectionDone, ConnectError,
-                           ConnectionLost, TCPTimedOutError, ResponseFailed,
-                           IOError)
+    DOWNLOAD_EXCEPTIONS = (
+        defer.TimeoutError,
+        TimeoutError,
+        DNSLookupError,
+        ConnectionRefusedError,
+        ConnectionDone,
+        ConnectError,
+        ConnectionLost,
+        TCPTimedOutError,
+        ResponseFailed,
+        IOError,
+    )
 
     def __init__(self, settings: Settings, stats: StatsCollector) -> None:
-        if not settings.getbool('HTTPCACHE_ENABLED'):
+        if not settings.getbool("HTTPCACHE_ENABLED"):
             raise NotConfigured
-        self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
-        self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
-        self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
+        self.policy = load_object(settings["HTTPCACHE_POLICY"])(settings)
+        self.storage = load_object(settings["HTTPCACHE_STORAGE"])(settings)
+        self.ignore_missing = settings.getbool("HTTPCACHE_IGNORE_MISSING")
         self.stats = stats
 
     @classmethod
-    def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
+    def from_crawler(
+        cls: Type[HttpCacheMiddlewareTV], crawler: Crawler
+    ) -> HttpCacheMiddlewareTV:
         o = cls(crawler.settings, crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@@ -56,78 +66,86 @@ class HttpCacheMiddleware:
         self.storage.close_spider(spider)
 
     def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
-        if request.meta.get('dont_cache', False):
+        if request.meta.get("dont_cache", False):
             return None
 
         # Skip uncacheable requests
         if not self.policy.should_cache_request(request):
-            request.meta['_dont_cache'] = True  # flag as uncacheable
+            request.meta["_dont_cache"] = True  # flag as uncacheable
             return None
 
         # Look for cached response and check if expired
         cachedresponse = self.storage.retrieve_response(spider, request)
         if cachedresponse is None:
-            self.stats.inc_value('httpcache/miss', spider=spider)
+            self.stats.inc_value("httpcache/miss", spider=spider)
             if self.ignore_missing:
-                self.stats.inc_value('httpcache/ignore', spider=spider)
+                self.stats.inc_value("httpcache/ignore", spider=spider)
                 raise IgnoreRequest(f"Ignored request not in cache: {request}")
             return None  # first time request
 
         # Return cached response only if not expired
-        cachedresponse.flags.append('cached')
+        cachedresponse.flags.append("cached")
         if self.policy.is_cached_response_fresh(cachedresponse, request):
-            self.stats.inc_value('httpcache/hit', spider=spider)
+            self.stats.inc_value("httpcache/hit", spider=spider)
             return cachedresponse
 
         # Keep a reference to cached response to avoid a second cache lookup on
         # process_response hook
-        request.meta['cached_response'] = cachedresponse
+        request.meta["cached_response"] = cachedresponse
 
         return None
 
-    def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
-        if request.meta.get('dont_cache', False):
+    def process_response(
+        self, request: Request, response: Response, spider: Spider
+    ) -> Response:
+        if request.meta.get("dont_cache", False):
             return response
 
         # Skip cached responses and uncacheable requests
-        if 'cached' in response.flags or '_dont_cache' in request.meta:
-            request.meta.pop('_dont_cache', None)
+        if "cached" in response.flags or "_dont_cache" in request.meta:
+            request.meta.pop("_dont_cache", None)
             return response
 
         # RFC2616 requires origin server to set Date header,
         # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
-        if 'Date' not in response.headers:
-            response.headers['Date'] = formatdate(usegmt=True)
+        if "Date" not in response.headers:
+            response.headers["Date"] = formatdate(usegmt=True)
 
         # Do not validate first-hand responses
-        cachedresponse = request.meta.pop('cached_response', None)
+        cachedresponse = request.meta.pop("cached_response", None)
         if cachedresponse is None:
-            self.stats.inc_value('httpcache/firsthand', spider=spider)
+            self.stats.inc_value("httpcache/firsthand", spider=spider)
             self._cache_response(spider, response, request, cachedresponse)
             return response
 
         if self.policy.is_cached_response_valid(cachedresponse, response, request):
-            self.stats.inc_value('httpcache/revalidate', spider=spider)
+            self.stats.inc_value("httpcache/revalidate", spider=spider)
             return cachedresponse
 
-        self.stats.inc_value('httpcache/invalidate', spider=spider)
+        self.stats.inc_value("httpcache/invalidate", spider=spider)
         self._cache_response(spider, response, request, cachedresponse)
         return response
 
     def process_exception(
         self, request: Request, exception: Exception, spider: Spider
     ) -> Optional[Response]:
-        cachedresponse = request.meta.pop('cached_response', None)
-        if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
-            self.stats.inc_value('httpcache/errorrecovery', spider=spider)
+        cachedresponse = request.meta.pop("cached_response", None)
+        if cachedresponse is not None and isinstance(
+            exception, self.DOWNLOAD_EXCEPTIONS
+        ):
+            self.stats.inc_value("httpcache/errorrecovery", spider=spider)
             return cachedresponse
         return None
 
     def _cache_response(
-        self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
+        self,
+        spider: Spider,
+        response: Response,
+        request: Request,
+        cachedresponse: Optional[Response],
     ) -> None:
         if self.policy.should_cache_response(response, request):
-            self.stats.inc_value('httpcache/store', spider=spider)
+            self.stats.inc_value("httpcache/store", spider=spider)
             self.storage.store_response(spider, request, response)
         else:
-            self.stats.inc_value('httpcache/uncacheable', spider=spider)
+            self.stats.inc_value("httpcache/uncacheable", spider=spider)
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index e57f6fcf8..7b5f4824a 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -8,17 +8,19 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.gz import gunzip
 
-ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
+ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
 
 try:
     import brotli
-    ACCEPTED_ENCODINGS.append(b'br')
+
+    ACCEPTED_ENCODINGS.append(b"br")
 except ImportError:
     pass
 
 try:
     import zstandard
-    ACCEPTED_ENCODINGS.append(b'zstd')
+
+    ACCEPTED_ENCODINGS.append(b"zstd")
 except ImportError:
     pass
 
@@ -26,12 +28,13 @@ except ImportError:
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
+
     def __init__(self, stats=None):
         self.stats = stats
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('COMPRESSION_ENABLED'):
+        if not crawler.settings.getbool("COMPRESSION_ENABLED"):
             raise NotConfigured
         try:
             return cls(stats=crawler.stats)
@@ -47,21 +50,26 @@ class HttpCompressionMiddleware:
             return result
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Encoding',
-                                   b", ".join(ACCEPTED_ENCODINGS))
+        request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
 
-        if request.method == 'HEAD':
+        if request.method == "HEAD":
             return response
         if isinstance(response, Response):
-            content_encoding = response.headers.getlist('Content-Encoding')
+            content_encoding = response.headers.getlist("Content-Encoding")
             if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
                 if self.stats:
-                    self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-                    self.stats.inc_value('httpcompression/response_count', spider=spider)
+                    self.stats.inc_value(
+                        "httpcompression/response_bytes",
+                        len(decoded_body),
+                        spider=spider,
+                    )
+                    self.stats.inc_value(
+                        "httpcompression/response_count", spider=spider
+                    )
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
@@ -69,18 +77,18 @@ class HttpCompressionMiddleware:
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable
-                    kwargs['encoding'] = None
+                    kwargs["encoding"] = None
                 response = response.replace(**kwargs)
                 if not content_encoding:
-                    del response.headers['Content-Encoding']
+                    del response.headers["Content-Encoding"]
 
         return response
 
     def _decode(self, body, encoding):
-        if encoding == b'gzip' or encoding == b'x-gzip':
+        if encoding == b"gzip" or encoding == b"x-gzip":
             body = gunzip(body)
 
-        if encoding == b'deflate':
+        if encoding == b"deflate":
             try:
                 body = zlib.decompress(body)
             except zlib.error:
@@ -90,9 +98,9 @@ class HttpCompressionMiddleware:
                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
                 body = zlib.decompress(body, -15)
-        if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
+        if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
             body = brotli.decompress(body)
-        if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
+        if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
             # Using its streaming API since its simple API could handle only cases
             # where there is content size data embedded in the frame
             reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index dd8a7e797..489867918 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -8,8 +8,7 @@ from scrapy.utils.python import to_bytes
 
 
 class HttpProxyMiddleware:
-
-    def __init__(self, auth_encoding='latin-1'):
+    def __init__(self, auth_encoding="latin-1"):
         self.auth_encoding = auth_encoding
         self.proxies = {}
         for type_, url in getproxies().items():
@@ -22,20 +21,20 @@ class HttpProxyMiddleware:
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
+        if not crawler.settings.getbool("HTTPPROXY_ENABLED"):
             raise NotConfigured
-        auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
+        auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
         return cls(auth_encoding)
 
     def _basic_auth_header(self, username, password):
         user_pass = to_bytes(
-            f'{unquote(username)}:{unquote(password)}',
-            encoding=self.auth_encoding)
+            f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding
+        )
         return base64.b64encode(user_pass)
 
     def _get_proxy(self, url, orig_type):
         proxy_type, user, password, hostport = _parse_proxy(url)
-        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
+        proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))
 
         if user:
             creds = self._basic_auth_header(user, password)
@@ -46,39 +45,36 @@ class HttpProxyMiddleware:
 
     def process_request(self, request, spider):
         creds, proxy_url = None, None
-        if 'proxy' in request.meta:
-            if request.meta['proxy'] is not None:
-                creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+        if "proxy" in request.meta:
+            if request.meta["proxy"] is not None:
+                creds, proxy_url = self._get_proxy(request.meta["proxy"], "")
         elif self.proxies:
             parsed = urlparse_cached(request)
             scheme = parsed.scheme
             if (
-                (
-                    # 'no_proxy' is only supported by http schemes
-                    scheme not in ('http', 'https')
-                    or not proxy_bypass(parsed.hostname)
-                )
-                and scheme in self.proxies
-            ):
+                # 'no_proxy' is only supported by http schemes
+                scheme not in ("http", "https")
+                or not proxy_bypass(parsed.hostname)
+            ) and scheme in self.proxies:
                 creds, proxy_url = self.proxies[scheme]
 
         self._set_proxy_and_creds(request, proxy_url, creds)
 
     def _set_proxy_and_creds(self, request, proxy_url, creds):
         if proxy_url:
-            request.meta['proxy'] = proxy_url
-        elif request.meta.get('proxy') is not None:
-            request.meta['proxy'] = None
+            request.meta["proxy"] = proxy_url
+        elif request.meta.get("proxy") is not None:
+            request.meta["proxy"] = None
         if creds:
-            request.headers[b'Proxy-Authorization'] = b'Basic ' + creds
-            request.meta['_auth_proxy'] = proxy_url
-        elif '_auth_proxy' in request.meta:
-            if proxy_url != request.meta['_auth_proxy']:
-                if b'Proxy-Authorization' in request.headers:
-                    del request.headers[b'Proxy-Authorization']
-                del request.meta['_auth_proxy']
-        elif b'Proxy-Authorization' in request.headers:
+            request.headers[b"Proxy-Authorization"] = b"Basic " + creds
+            request.meta["_auth_proxy"] = proxy_url
+        elif "_auth_proxy" in request.meta:
+            if proxy_url != request.meta["_auth_proxy"]:
+                if b"Proxy-Authorization" in request.headers:
+                    del request.headers[b"Proxy-Authorization"]
+                del request.meta["_auth_proxy"]
+        elif b"Proxy-Authorization" in request.headers:
             if proxy_url:
-                request.meta['_auth_proxy'] = proxy_url
+                request.meta["_auth_proxy"] = proxy_url
             else:
-                del request.headers[b'Proxy-Authorization']
+                del request.headers[b"Proxy-Authorization"]
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index fba825947..003c59fc4 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -17,57 +17,66 @@ def _build_redirect_request(source_request, *, url, **kwargs):
         **kwargs,
         cookies=None,
     )
-    if 'Cookie' in redirect_request.headers:
+    if "Cookie" in redirect_request.headers:
         source_request_netloc = urlparse_cached(source_request).netloc
         redirect_request_netloc = urlparse_cached(redirect_request).netloc
         if source_request_netloc != redirect_request_netloc:
-            del redirect_request.headers['Cookie']
+            del redirect_request.headers["Cookie"]
     return redirect_request
 
 
 class BaseRedirectMiddleware:
 
-    enabled_setting = 'REDIRECT_ENABLED'
+    enabled_setting = "REDIRECT_ENABLED"
 
     def __init__(self, settings):
         if not settings.getbool(self.enabled_setting):
             raise NotConfigured
 
-        self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
-        self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
+        self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES")
+        self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST")
 
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
 
     def _redirect(self, redirected, request, spider, reason):
-        ttl = request.meta.setdefault('redirect_ttl', self.max_redirect_times)
-        redirects = request.meta.get('redirect_times', 0) + 1
+        ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
+        redirects = request.meta.get("redirect_times", 0) + 1
 
         if ttl and redirects <= self.max_redirect_times:
-            redirected.meta['redirect_times'] = redirects
-            redirected.meta['redirect_ttl'] = ttl - 1
-            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
-            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
+            redirected.meta["redirect_times"] = redirects
+            redirected.meta["redirect_ttl"] = ttl - 1
+            redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [
+                request.url
+            ]
+            redirected.meta["redirect_reasons"] = request.meta.get(
+                "redirect_reasons", []
+            ) + [reason]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
-            logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
-                         {'reason': reason, 'redirected': redirected, 'request': request},
-                         extra={'spider': spider})
+            logger.debug(
+                "Redirecting (%(reason)s) to %(redirected)s from %(request)s",
+                {"reason": reason, "redirected": redirected, "request": request},
+                extra={"spider": spider},
+            )
             return redirected
-        logger.debug("Discarding %(request)s: max redirections reached",
-                     {'request': request}, extra={'spider': spider})
+        logger.debug(
+            "Discarding %(request)s: max redirections reached",
+            {"request": request},
+            extra={"spider": spider},
+        )
         raise IgnoreRequest("max redirections reached")
 
     def _redirect_request_using_get(self, request, redirect_url):
         redirect_request = _build_redirect_request(
             request,
             url=redirect_url,
-            method='GET',
-            body='',
+            method="GET",
+            body="",
         )
-        redirect_request.headers.pop('Content-Type', None)
-        redirect_request.headers.pop('Content-Length', None)
+        redirect_request.headers.pop("Content-Type", None)
+        redirect_request.headers.pop("Content-Length", None)
         return redirect_request
 
 
@@ -79,25 +88,25 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
     def process_response(self, request, response, spider):
         if (
-            request.meta.get('dont_redirect', False)
-            or response.status in getattr(spider, 'handle_httpstatus_list', [])
-            or response.status in request.meta.get('handle_httpstatus_list', [])
-            or request.meta.get('handle_httpstatus_all', False)
+            request.meta.get("dont_redirect", False)
+            or response.status in getattr(spider, "handle_httpstatus_list", [])
+            or response.status in request.meta.get("handle_httpstatus_list", [])
+            or request.meta.get("handle_httpstatus_all", False)
         ):
             return response
 
         allowed_status = (301, 302, 303, 307, 308)
-        if 'Location' not in response.headers or response.status not in allowed_status:
+        if "Location" not in response.headers or response.status not in allowed_status:
             return response
 
-        location = safe_url_string(response.headers['Location'])
-        if response.headers['Location'].startswith(b'//'):
+        location = safe_url_string(response.headers["Location"])
+        if response.headers["Location"].startswith(b"//"):
             request_scheme = urlparse(request.url).scheme
-            location = request_scheme + '://' + location.lstrip('/')
+            location = request_scheme + "://" + location.lstrip("/")
 
         redirected_url = urljoin(request.url, location)
 
-        if response.status in (301, 307, 308) or request.method == 'HEAD':
+        if response.status in (301, 307, 308) or request.method == "HEAD":
             redirected = _build_redirect_request(request, url=redirected_url)
             return self._redirect(redirected, request, spider, response.status)
 
@@ -107,25 +116,24 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):
 
-    enabled_setting = 'METAREFRESH_ENABLED'
+    enabled_setting = "METAREFRESH_ENABLED"
 
     def __init__(self, settings):
         super().__init__(settings)
-        self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
-        self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
+        self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS")
+        self._maxdelay = settings.getint("METAREFRESH_MAXDELAY")
 
     def process_response(self, request, response, spider):
         if (
-            request.meta.get('dont_redirect', False)
-            or request.method == 'HEAD'
+            request.meta.get("dont_redirect", False)
+            or request.method == "HEAD"
             or not isinstance(response, HtmlResponse)
         ):
             return response
 
-        interval, url = get_meta_refresh(response,
-                                         ignore_tags=self._ignore_tags)
+        interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
         if url and interval < self._maxdelay:
             redirected = self._redirect_request_using_get(request, url)
-            return self._redirect(redirected, request, spider, 'meta refresh')
+            return self._redirect(redirected, request, spider, "meta refresh")
 
         return response
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 2de610e86..8a8f15f9a 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -39,11 +39,11 @@ def get_retry_request(
     request: Request,
     *,
     spider: Spider,
-    reason: Union[str, Exception] = 'unspecified',
+    reason: Union[str, Exception] = "unspecified",
     max_retry_times: Optional[int] = None,
     priority_adjust: Optional[int] = None,
     logger: Logger = retry_logger,
-    stats_base_key: str = 'retry',
+    stats_base_key: str = "retry",
 ):
     """
     Returns a new :class:`~scrapy.Request` object to retry the specified
@@ -87,22 +87,22 @@ def get_retry_request(
     """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
-    retry_times = request.meta.get('retry_times', 0) + 1
+    retry_times = request.meta.get("retry_times", 0) + 1
     if max_retry_times is None:
-        max_retry_times = request.meta.get('max_retry_times')
+        max_retry_times = request.meta.get("max_retry_times")
         if max_retry_times is None:
-            max_retry_times = settings.getint('RETRY_TIMES')
+            max_retry_times = settings.getint("RETRY_TIMES")
     if retry_times <= max_retry_times:
         logger.debug(
             "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
-            {'request': request, 'retry_times': retry_times, 'reason': reason},
-            extra={'spider': spider}
+            {"request": request, "retry_times": retry_times, "reason": reason},
+            extra={"spider": spider},
         )
         new_request: Request = request.copy()
-        new_request.meta['retry_times'] = retry_times
+        new_request.meta["retry_times"] = retry_times
         new_request.dont_filter = True
         if priority_adjust is None:
-            priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
+            priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
         new_request.priority = request.priority + priority_adjust
 
         if callable(reason):
@@ -110,15 +110,14 @@ def get_retry_request(
         if isinstance(reason, Exception):
             reason = global_object_name(reason.__class__)
 
-        stats.inc_value(f'{stats_base_key}/count')
-        stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
+        stats.inc_value(f"{stats_base_key}/count")
+        stats.inc_value(f"{stats_base_key}/reason_count/{reason}")
         return new_request
-    stats.inc_value(f'{stats_base_key}/max_reached')
+    stats.inc_value(f"{stats_base_key}/max_reached")
     logger.error(
-        "Gave up retrying %(request)s (failed %(retry_times)d times): "
-        "%(reason)s",
-        {'request': request, 'retry_times': retry_times, 'reason': reason},
-        extra={'spider': spider},
+        "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s",
+        {"request": request, "retry_times": retry_times, "reason": reason},
+        extra={"spider": spider},
     )
     return None
 
@@ -127,24 +126,35 @@ class RetryMiddleware:
 
     # IOError is raised by the HttpCompression middleware when trying to
     # decompress an empty response
-    EXCEPTIONS_TO_RETRY = (defer.TimeoutError, TimeoutError, DNSLookupError,
-                           ConnectionRefusedError, ConnectionDone, ConnectError,
-                           ConnectionLost, TCPTimedOutError, ResponseFailed,
-                           IOError, TunnelError)
+    EXCEPTIONS_TO_RETRY = (
+        defer.TimeoutError,
+        TimeoutError,
+        DNSLookupError,
+        ConnectionRefusedError,
+        ConnectionDone,
+        ConnectError,
+        ConnectionLost,
+        TCPTimedOutError,
+        ResponseFailed,
+        IOError,
+        TunnelError,
+    )
 
     def __init__(self, settings):
-        if not settings.getbool('RETRY_ENABLED'):
+        if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
-        self.max_retry_times = settings.getint('RETRY_TIMES')
-        self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
-        self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
+        self.max_retry_times = settings.getint("RETRY_TIMES")
+        self.retry_http_codes = set(
+            int(x) for x in settings.getlist("RETRY_HTTP_CODES")
+        )
+        self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
 
     def process_response(self, request, response, spider):
-        if request.meta.get('dont_retry', False):
+        if request.meta.get("dont_retry", False):
             return response
         if response.status in self.retry_http_codes:
             reason = response_status_message(response.status)
@@ -152,15 +162,14 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if (
-            isinstance(exception, self.EXCEPTIONS_TO_RETRY)
-            and not request.meta.get('dont_retry', False)
+        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
+            "dont_retry", False
         ):
             return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):
-        max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
-        priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
+        max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
+        priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
         return get_retry_request(
             request,
             reason=reason,
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 7bd39aa43..326c35290 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -20,23 +20,23 @@ class RobotsTxtMiddleware:
     DOWNLOAD_PRIORITY = 1000
 
     def __init__(self, crawler):
-        if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
+        if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
             raise NotConfigured
-        self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
-        self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
+        self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy")
+        self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None)
         self.crawler = crawler
         self._parsers = {}
-        self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
+        self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER"))
 
         # check if parser dependencies are met, this should throw an error otherwise.
-        self._parserimpl.from_crawler(self.crawler, b'')
+        self._parserimpl.from_crawler(self.crawler, b"")
 
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler)
 
     def process_request(self, request, spider):
-        if request.meta.get('dont_obey_robotstxt'):
+        if request.meta.get("dont_obey_robotstxt"):
             return
         d = maybeDeferred(self.robot_parser, request, spider)
         d.addCallback(self.process_request_2, request, spider)
@@ -48,11 +48,14 @@ class RobotsTxtMiddleware:
 
         useragent = self._robotstxt_useragent
         if not useragent:
-            useragent = request.headers.get(b'User-Agent', self._default_useragent)
+            useragent = request.headers.get(b"User-Agent", self._default_useragent)
         if not rp.allowed(request.url, useragent):
-            logger.debug("Forbidden by robots.txt: %(request)s",
-                         {'request': request}, extra={'spider': spider})
-            self.crawler.stats.inc_value('robotstxt/forbidden')
+            logger.debug(
+                "Forbidden by robots.txt: %(request)s",
+                {"request": request},
+                extra={"spider": spider},
+            )
+            self.crawler.stats.inc_value("robotstxt/forbidden")
             raise IgnoreRequest("Forbidden by robots.txt")
 
     def robot_parser(self, request, spider):
@@ -65,13 +68,13 @@ class RobotsTxtMiddleware:
             robotsreq = Request(
                 robotsurl,
                 priority=self.DOWNLOAD_PRIORITY,
-                meta={'dont_obey_robotstxt': True}
+                meta={"dont_obey_robotstxt": True},
             )
             dfd = self.crawler.engine.download(robotsreq)
             dfd.addCallback(self._parse_robots, netloc, spider)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
-            self.crawler.stats.inc_value('robotstxt/request_count')
+            self.crawler.stats.inc_value("robotstxt/request_count")
 
         if isinstance(self._parsers[netloc], Deferred):
             d = Deferred()
@@ -79,21 +82,26 @@ class RobotsTxtMiddleware:
             def cb(result):
                 d.callback(result)
                 return result
+
             self._parsers[netloc].addCallback(cb)
             return d
         return self._parsers[netloc]
 
     def _logerror(self, failure, request, spider):
         if failure.type is not IgnoreRequest:
-            logger.error("Error downloading %(request)s: %(f_exception)s",
-                         {'request': request, 'f_exception': failure.value},
-                         exc_info=failure_to_exc_info(failure),
-                         extra={'spider': spider})
+            logger.error(
+                "Error downloading %(request)s: %(f_exception)s",
+                {"request": request, "f_exception": failure.value},
+                exc_info=failure_to_exc_info(failure),
+                extra={"spider": spider},
+            )
         return failure
 
     def _parse_robots(self, response, netloc, spider):
-        self.crawler.stats.inc_value('robotstxt/response_count')
-        self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}')
+        self.crawler.stats.inc_value("robotstxt/response_count")
+        self.crawler.stats.inc_value(
+            f"robotstxt/response_status_count/{response.status}"
+        )
         rp = self._parserimpl.from_crawler(self.crawler, response.body)
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = rp
@@ -101,7 +109,7 @@ class RobotsTxtMiddleware:
 
     def _robots_error(self, failure, netloc):
         if failure.type is not IgnoreRequest:
-            key = f'robotstxt/exception_count/{failure.type}'
+            key = f"robotstxt/exception_count/{failure.type}"
             self.crawler.stats.inc_value(key)
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = None
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 2bb022c16..a0f62e262 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -11,40 +11,50 @@ def get_header_size(headers):
         if isinstance(value, (list, tuple)):
             for v in value:
                 size += len(b": ") + len(key) + len(v)
-    return size + len(b'\r\n') * (len(headers.keys()) - 1)
+    return size + len(b"\r\n") * (len(headers.keys()) - 1)
 
 
 def get_status_size(response_status):
-    return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
+    return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15
     # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
 
 
 class DownloaderStats:
-
     def __init__(self, stats):
         self.stats = stats
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('DOWNLOADER_STATS'):
+        if not crawler.settings.getbool("DOWNLOADER_STATS"):
             raise NotConfigured
         return cls(crawler.stats)
 
     def process_request(self, request, spider):
-        self.stats.inc_value('downloader/request_count', spider=spider)
-        self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
+        self.stats.inc_value("downloader/request_count", spider=spider)
+        self.stats.inc_value(
+            f"downloader/request_method_count/{request.method}", spider=spider
+        )
         reqlen = len(request_httprepr(request))
-        self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
+        self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
 
     def process_response(self, request, response, spider):
-        self.stats.inc_value('downloader/response_count', spider=spider)
-        self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
-        reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
+        self.stats.inc_value("downloader/response_count", spider=spider)
+        self.stats.inc_value(
+            f"downloader/response_status_count/{response.status}", spider=spider
+        )
+        reslen = (
+            len(response.body)
+            + get_header_size(response.headers)
+            + get_status_size(response.status)
+            + 4
+        )
         # response.body + b"\r\n"+ response.header + b"\r\n" + response.status
-        self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
+        self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
         return response
 
     def process_exception(self, request, exception, spider):
         ex_class = global_object_name(exception.__class__)
-        self.stats.inc_value('downloader/exception_count', spider=spider)
-        self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)
+        self.stats.inc_value("downloader/exception_count", spider=spider)
+        self.stats.inc_value(
+            f"downloader/exception_type_count/{ex_class}", spider=spider
+        )
diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py
index 3ee7bd129..856a275ab 100644
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@@ -6,18 +6,18 @@ from scrapy import signals
 class UserAgentMiddleware:
     """This middleware allows spiders to override the user_agent"""
 
-    def __init__(self, user_agent='Scrapy'):
+    def __init__(self, user_agent="Scrapy"):
         self.user_agent = user_agent
 
     @classmethod
     def from_crawler(cls, crawler):
-        o = cls(crawler.settings['USER_AGENT'])
+        o = cls(crawler.settings["USER_AGENT"])
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         return o
 
     def spider_opened(self, spider):
-        self.user_agent = getattr(spider, 'user_agent', self.user_agent)
+        self.user_agent = getattr(spider, "user_agent", self.user_agent)
 
     def process_request(self, request, spider):
         if self.user_agent:
-            request.headers.setdefault(b'User-Agent', self.user_agent)
+            request.headers.setdefault(b"User-Agent", self.user_agent)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 6dca11d31..fa0f8f846 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -18,7 +18,9 @@ BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
 
 class BaseDupeFilter:
     @classmethod
-    def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV:
+    def from_settings(
+        cls: Type[BaseDupeFilterTV], settings: BaseSettings
+    ) -> BaseDupeFilterTV:
         return cls()
 
     def request_seen(self, request: Request) -> bool:
@@ -55,13 +57,15 @@ class RFPDupeFilter(BaseDupeFilter):
         self.debug = debug
         self.logger = logging.getLogger(__name__)
         if path:
-            self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8")
+            self.file = Path(path, "requests.seen").open("a+", encoding="utf-8")
             self.file.seek(0)
             self.fingerprints.update(x.rstrip() for x in self.file)
 
     @classmethod
-    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
-        debug = settings.getbool('DUPEFILTER_DEBUG')
+    def from_settings(
+        cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None
+    ) -> RFPDupeFilterTV:
+        debug = settings.getbool("DUPEFILTER_DEBUG")
         try:
             return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
         except TypeError:
@@ -100,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
             return True
         self.fingerprints.add(fp)
         if self.file:
-            self.file.write(fp + '\n')
+            self.file.write(fp + "\n")
         return False
 
     def request_fingerprint(self, request: Request) -> str:
@@ -113,13 +117,15 @@ class RFPDupeFilter(BaseDupeFilter):
     def log(self, request: Request, spider: Spider) -> None:
         if self.debug:
             msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
-            args = {'request': request, 'referer': referer_str(request)}
-            self.logger.debug(msg, args, extra={'spider': spider})
+            args = {"request": request, "referer": referer_str(request)}
+            self.logger.debug(msg, args, extra={"spider": spider})
         elif self.logdupes:
-            msg = ("Filtered duplicate request: %(request)s"
-                   " - no more duplicates will be shown"
-                   " (see DUPEFILTER_DEBUG to show all duplicates)")
-            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
+            msg = (
+                "Filtered duplicate request: %(request)s"
+                " - no more duplicates will be shown"
+                " (see DUPEFILTER_DEBUG to show all duplicates)"
+            )
+            self.logger.debug(msg, {"request": request}, extra={"spider": spider})
             self.logdupes = False
 
-        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
+        spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider)
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 0c410f035..7f202b8b8 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -10,6 +10,7 @@ new exceptions here without documenting them there.
 
 class NotConfigured(Exception):
     """Indicates a missing configuration situation"""
+
     pass
 
 
@@ -18,6 +19,7 @@ class _InvalidOutput(TypeError):
     Indicates an invalid value has been returned by a middleware's processing method.
     Internal and undocumented, it should not be raised or caught by user code.
     """
+
     pass
 
 
@@ -30,13 +32,14 @@ class IgnoreRequest(Exception):
 
 class DontCloseSpider(Exception):
     """Request the spider not to be closed yet"""
+
     pass
 
 
 class CloseSpider(Exception):
     """Raise this from callbacks to request the spider to be closed"""
 
-    def __init__(self, reason='cancelled'):
+    def __init__(self, reason="cancelled"):
         super().__init__()
         self.reason = reason
 
@@ -58,11 +61,13 @@ class StopDownload(Exception):
 
 class DropItem(Exception):
     """Drop item from the item pipeline"""
+
     pass
 
 
 class NotSupported(Exception):
     """Indicates a feature or method is not supported"""
+
     pass
 
 
@@ -73,7 +78,7 @@ class UsageError(Exception):
     """To indicate a command-line usage error"""
 
     def __init__(self, *a, **kw):
-        self.print_help = kw.pop('print_help', True)
+        self.print_help = kw.pop("print_help", True)
         super().__init__(*a, **kw)
 
 
@@ -81,9 +86,11 @@ class ScrapyDeprecationWarning(Warning):
     """Warning category for deprecated features, since the default
     DeprecationWarning is silenced on Python 2.7+
     """
+
     pass
 
 
 class ContractFail(AssertionError):
     """Error raised in case of a failing contract"""
+
     pass
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2135c3bbc..7d9a9b6ff 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -19,13 +19,19 @@ from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
-__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
-           'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',
-           'JsonItemExporter', 'MarshalItemExporter']
+__all__ = [
+    "BaseItemExporter",
+    "PprintItemExporter",
+    "PickleItemExporter",
+    "CsvItemExporter",
+    "XmlItemExporter",
+    "JsonLinesItemExporter",
+    "JsonItemExporter",
+    "MarshalItemExporter",
+]
 
 
 class BaseItemExporter:
-
     def __init__(self, *, dont_fail=False, **kwargs):
         self._kwargs = kwargs
         self._configure(kwargs, dont_fail=dont_fail)
@@ -35,10 +41,10 @@ class BaseItemExporter:
         If dont_fail is set, it won't raise an exception on unexpected options
         (useful for using with keyword arguments in subclasses ``__init__`` methods)
         """
-        self.encoding = options.pop('encoding', None)
-        self.fields_to_export = options.pop('fields_to_export', None)
-        self.export_empty_fields = options.pop('export_empty_fields', False)
-        self.indent = options.pop('indent', None)
+        self.encoding = options.pop("encoding", None)
+        self.fields_to_export = options.pop("fields_to_export", None)
+        self.export_empty_fields = options.pop("export_empty_fields", False)
+        self.indent = options.pop("indent", None)
         if not dont_fail and options:
             raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
 
@@ -46,7 +52,7 @@ class BaseItemExporter:
         raise NotImplementedError
 
     def serialize_field(self, field, name, value):
-        serializer = field.get('serializer', lambda x: x)
+        serializer = field.get("serializer", lambda x: x)
         return serializer(value)
 
     def start_exporting(self):
@@ -74,8 +80,7 @@ class BaseItemExporter:
                 field_iter = self.fields_to_export.items()
             else:
                 field_iter = (
-                    (x, y) for x, y in self.fields_to_export.items()
-                    if x in item
+                    (x, y) for x, y in self.fields_to_export.items() if x in item
                 )
         else:
             if include_empty:
@@ -98,36 +103,36 @@ class BaseItemExporter:
 
 
 class JsonLinesItemExporter(BaseItemExporter):
-
     def __init__(self, file, **kwargs):
         super().__init__(dont_fail=True, **kwargs)
         self.file = file
-        self._kwargs.setdefault('ensure_ascii', not self.encoding)
+        self._kwargs.setdefault("ensure_ascii", not self.encoding)
         self.encoder = ScrapyJSONEncoder(**self._kwargs)
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
-        data = self.encoder.encode(itemdict) + '\n'
+        data = self.encoder.encode(itemdict) + "\n"
         self.file.write(to_bytes(data, self.encoding))
 
 
 class JsonItemExporter(BaseItemExporter):
-
     def __init__(self, file, **kwargs):
         super().__init__(dont_fail=True, **kwargs)
         self.file = file
         # there is a small difference between the behaviour or JsonItemExporter.indent
         # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
         # the addition of newlines everywhere
-        json_indent = self.indent if self.indent is not None and self.indent > 0 else None
-        self._kwargs.setdefault('indent', json_indent)
-        self._kwargs.setdefault('ensure_ascii', not self.encoding)
+        json_indent = (
+            self.indent if self.indent is not None and self.indent > 0 else None
+        )
+        self._kwargs.setdefault("indent", json_indent)
+        self._kwargs.setdefault("ensure_ascii", not self.encoding)
         self.encoder = ScrapyJSONEncoder(**self._kwargs)
         self.first_item = True
 
     def _beautify_newline(self):
         if self.indent is not None:
-            self.file.write(b'\n')
+            self.file.write(b"\n")
 
     def start_exporting(self):
         self.file.write(b"[")
@@ -141,7 +146,7 @@ class JsonItemExporter(BaseItemExporter):
         if self.first_item:
             self.first_item = False
         else:
-            self.file.write(b',')
+            self.file.write(b",")
             self._beautify_newline()
         itemdict = dict(self._get_serialized_fields(item))
         data = self.encoder.encode(itemdict)
@@ -149,22 +154,21 @@ class JsonItemExporter(BaseItemExporter):
 
 
 class XmlItemExporter(BaseItemExporter):
-
     def __init__(self, file, **kwargs):
-        self.item_element = kwargs.pop('item_element', 'item')
-        self.root_element = kwargs.pop('root_element', 'items')
+        self.item_element = kwargs.pop("item_element", "item")
+        self.root_element = kwargs.pop("root_element", "items")
         super().__init__(**kwargs)
         if not self.encoding:
-            self.encoding = 'utf-8'
+            self.encoding = "utf-8"
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
     def _beautify_newline(self, new_item=False):
         if self.indent is not None and (self.indent > 0 or new_item):
-            self.xg.characters('\n')
+            self.xg.characters("\n")
 
     def _beautify_indent(self, depth=1):
         if self.indent:
-            self.xg.characters(' ' * self.indent * depth)
+            self.xg.characters(" " * self.indent * depth)
 
     def start_exporting(self):
         self.xg.startDocument()
@@ -175,7 +179,7 @@ class XmlItemExporter(BaseItemExporter):
         self._beautify_indent(depth=1)
         self.xg.startElement(self.item_element, {})
         self._beautify_newline()
-        for name, value in self._get_serialized_fields(item, default_value=''):
+        for name, value in self._get_serialized_fields(item, default_value=""):
             self._export_xml_field(name, value, depth=2)
         self._beautify_indent(depth=1)
         self.xg.endElement(self.item_element)
@@ -188,7 +192,7 @@ class XmlItemExporter(BaseItemExporter):
     def _export_xml_field(self, name, serialized_value, depth):
         self._beautify_indent(depth=depth)
         self.xg.startElement(name, {})
-        if hasattr(serialized_value, 'items'):
+        if hasattr(serialized_value, "items"):
             self._beautify_newline()
             for subname, value in serialized_value.items():
                 self._export_xml_field(subname, value, depth=depth + 1)
@@ -196,7 +200,7 @@ class XmlItemExporter(BaseItemExporter):
         elif is_listlike(serialized_value):
             self._beautify_newline()
             for value in serialized_value:
-                self._export_xml_field('value', value, depth=depth + 1)
+                self._export_xml_field("value", value, depth=depth + 1)
             self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, str):
             self.xg.characters(serialized_value)
@@ -207,18 +211,24 @@ class XmlItemExporter(BaseItemExporter):
 
 
 class CsvItemExporter(BaseItemExporter):
-
-    def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
+    def __init__(
+        self,
+        file,
+        include_headers_line=True,
+        join_multivalued=",",
+        errors=None,
+        **kwargs,
+    ):
         super().__init__(dont_fail=True, **kwargs)
         if not self.encoding:
-            self.encoding = 'utf-8'
+            self.encoding = "utf-8"
         self.include_headers_line = include_headers_line
         self.stream = io.TextIOWrapper(
             file,
             line_buffering=False,
             write_through=True,
             encoding=self.encoding,
-            newline='',  # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+            newline="",  # Windows needs this https://github.com/scrapy/scrapy/issues/3034
             errors=errors,
         )
         self.csv_writer = csv.writer(self.stream, **self._kwargs)
@@ -226,7 +236,7 @@ class CsvItemExporter(BaseItemExporter):
         self._join_multivalued = join_multivalued
 
     def serialize_field(self, field, name, value):
-        serializer = field.get('serializer', self._join_if_needed)
+        serializer = field.get("serializer", self._join_if_needed)
         return serializer(value)
 
     def _join_if_needed(self, value):
@@ -242,8 +252,7 @@ class CsvItemExporter(BaseItemExporter):
             self._headers_not_written = False
             self._write_headers_and_set_fields_to_export(item)
 
-        fields = self._get_serialized_fields(item, default_value='',
-                                             include_empty=True)
+        fields = self._get_serialized_fields(item, default_value="", include_empty=True)
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
@@ -268,7 +277,6 @@ class CsvItemExporter(BaseItemExporter):
 
 
 class PickleItemExporter(BaseItemExporter):
-
     def __init__(self, file, protocol=4, **kwargs):
         super().__init__(**kwargs)
         self.file = file
@@ -297,14 +305,13 @@ class MarshalItemExporter(BaseItemExporter):
 
 
 class PprintItemExporter(BaseItemExporter):
-
     def __init__(self, file, **kwargs):
         super().__init__(**kwargs)
         self.file = file
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))
+        self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
 
 
 class PythonItemExporter(BaseItemExporter):
@@ -318,17 +325,18 @@ class PythonItemExporter(BaseItemExporter):
     """
 
     def _configure(self, options, dont_fail=False):
-        self.binary = options.pop('binary', True)
+        self.binary = options.pop("binary", True)
         super()._configure(options, dont_fail)
         if self.binary:
             warnings.warn(
                 "PythonItemExporter will drop support for binary export in the future",
-                ScrapyDeprecationWarning)
+                ScrapyDeprecationWarning,
+            )
         if not self.encoding:
-            self.encoding = 'utf-8'
+            self.encoding = "utf-8"
 
     def serialize_field(self, field, name, value):
-        serializer = field.get('serializer', self._serialize_value)
+        serializer = field.get("serializer", self._serialize_value)
         return serializer(value)
 
     def _serialize_value(self, value):
diff --git a/scrapy/extension.py b/scrapy/extension.py
index 050b87e5f..89f6a694d 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -9,8 +9,8 @@ from scrapy.utils.conf import build_component_list
 
 class ExtensionManager(MiddlewareManager):
 
-    component_name = 'extension'
+    component_name = "extension"
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings.getwithbase('EXTENSIONS'))
+        return build_component_list(settings.getwithbase("EXTENSIONS"))
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 812844c0a..bb6f832f2 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -11,15 +11,14 @@ from scrapy.exceptions import NotConfigured
 
 
 class CloseSpider:
-
     def __init__(self, crawler):
         self.crawler = crawler
 
         self.close_on = {
-            'timeout': crawler.settings.getfloat('CLOSESPIDER_TIMEOUT'),
-            'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
-            'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
-            'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
+            "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"),
+            "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
+            "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
+            "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
         }
 
         if not any(self.close_on.values()):
@@ -27,13 +26,13 @@ class CloseSpider:
 
         self.counter = defaultdict(int)
 
-        if self.close_on.get('errorcount'):
+        if self.close_on.get("errorcount"):
             crawler.signals.connect(self.error_count, signal=signals.spider_error)
-        if self.close_on.get('pagecount'):
+        if self.close_on.get("pagecount"):
             crawler.signals.connect(self.page_count, signal=signals.response_received)
-        if self.close_on.get('timeout'):
+        if self.close_on.get("timeout"):
             crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
-        if self.close_on.get('itemcount'):
+        if self.close_on.get("itemcount"):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
@@ -42,27 +41,31 @@ class CloseSpider:
         return cls(crawler)
 
     def error_count(self, failure, response, spider):
-        self.counter['errorcount'] += 1
-        if self.counter['errorcount'] == self.close_on['errorcount']:
-            self.crawler.engine.close_spider(spider, 'closespider_errorcount')
+        self.counter["errorcount"] += 1
+        if self.counter["errorcount"] == self.close_on["errorcount"]:
+            self.crawler.engine.close_spider(spider, "closespider_errorcount")
 
     def page_count(self, response, request, spider):
-        self.counter['pagecount'] += 1
-        if self.counter['pagecount'] == self.close_on['pagecount']:
-            self.crawler.engine.close_spider(spider, 'closespider_pagecount')
+        self.counter["pagecount"] += 1
+        if self.counter["pagecount"] == self.close_on["pagecount"]:
+            self.crawler.engine.close_spider(spider, "closespider_pagecount")
 
     def spider_opened(self, spider):
         from twisted.internet import reactor
-        self.task = reactor.callLater(self.close_on['timeout'],
-                                      self.crawler.engine.close_spider, spider,
-                                      reason='closespider_timeout')
+
+        self.task = reactor.callLater(
+            self.close_on["timeout"],
+            self.crawler.engine.close_spider,
+            spider,
+            reason="closespider_timeout",
+        )
 
     def item_scraped(self, item, spider):
-        self.counter['itemcount'] += 1
-        if self.counter['itemcount'] == self.close_on['itemcount']:
-            self.crawler.engine.close_spider(spider, 'closespider_itemcount')
+        self.counter["itemcount"] += 1
+        if self.counter["itemcount"] == self.close_on["itemcount"]:
+            self.crawler.engine.close_spider(spider, "closespider_itemcount")
 
     def spider_closed(self, spider):
-        task = getattr(self, 'task', False)
+        task = getattr(self, "task", False)
         if task and task.active():
             task.cancel()
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 675f8276f..30c987253 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -7,7 +7,6 @@ from scrapy import signals
 
 
 class CoreStats:
-
     def __init__(self, stats):
         self.stats = stats
         self.start_time = None
@@ -24,23 +23,25 @@ class CoreStats:
 
     def spider_opened(self, spider):
         self.start_time = datetime.utcnow()
-        self.stats.set_value('start_time', self.start_time, spider=spider)
+        self.stats.set_value("start_time", self.start_time, spider=spider)
 
     def spider_closed(self, spider, reason):
         finish_time = datetime.utcnow()
         elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
-        self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
-        self.stats.set_value('finish_time', finish_time, spider=spider)
-        self.stats.set_value('finish_reason', reason, spider=spider)
+        self.stats.set_value(
+            "elapsed_time_seconds", elapsed_time_seconds, spider=spider
+        )
+        self.stats.set_value("finish_time", finish_time, spider=spider)
+        self.stats.set_value("finish_reason", reason, spider=spider)
 
     def item_scraped(self, item, spider):
-        self.stats.inc_value('item_scraped_count', spider=spider)
+        self.stats.inc_value("item_scraped_count", spider=spider)
 
     def response_received(self, spider):
-        self.stats.inc_value('response_received_count', spider=spider)
+        self.stats.inc_value("response_received_count", spider=spider)
 
     def item_dropped(self, item, spider, exception):
         reason = exception.__class__.__name__
-        self.stats.inc_value('item_dropped_count', spider=spider)
-        self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)
+        self.stats.inc_value("item_dropped_count", spider=spider)
+        self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index fd2a02d8d..8628b4a1e 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -18,7 +18,6 @@ logger = logging.getLogger(__name__)
 
 
 class StackTraceDump:
-
     def __init__(self, crawler=None):
         self.crawler = crawler
         try:
@@ -34,20 +33,23 @@ class StackTraceDump:
 
     def dump_stacktrace(self, signum, frame):
         log_args = {
-            'stackdumps': self._thread_stacks(),
-            'enginestatus': format_engine_status(self.crawler.engine),
-            'liverefs': format_live_refs(),
+            "stackdumps": self._thread_stacks(),
+            "enginestatus": format_engine_status(self.crawler.engine),
+            "liverefs": format_live_refs(),
         }
-        logger.info("Dumping stack trace and engine status\n"
-                    "%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s",
-                    log_args, extra={'crawler': self.crawler})
+        logger.info(
+            "Dumping stack trace and engine status\n"
+            "%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s",
+            log_args,
+            extra={"crawler": self.crawler},
+        )
 
     def _thread_stacks(self):
         id2name = dict((th.ident, th.name) for th in threading.enumerate())
-        dumps = ''
+        dumps = ""
         for id_, frame in sys._current_frames().items():
-            name = id2name.get(id_, '')
-            dump = ''.join(traceback.format_stack(frame))
+            name = id2name.get(id_, "")
+            dump = "".join(traceback.format_stack(frame))
             dumps += f"# Thread: {name}({id_})\n{dump}\n"
         return dumps
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0aa27e417..ca4d77a35 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -34,15 +34,15 @@ logger = logging.getLogger(__name__)
 
 def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
     argument_names = get_func_args(builder)
-    if 'feed_options' in argument_names:
-        kwargs['feed_options'] = feed_options
+    if "feed_options" in argument_names:
+        kwargs["feed_options"] = feed_options
     else:
         warnings.warn(
             f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
             "'feed_options' parameter to its signature to remove this "
             "warning. This parameter will become mandatory in a future "
             "version of Scrapy.",
-            category=ScrapyDeprecationWarning
+            category=ScrapyDeprecationWarning,
         )
     return builder(*preargs, uri, *args, **kwargs)
 
@@ -55,6 +55,7 @@ class ItemFilter:
     :param feed_options: feed specific options passed from FeedExporter
     :type feed_options: dict
     """
+
     feed_options: Optional[dict]
     item_classes: Tuple
 
@@ -62,7 +63,8 @@ class ItemFilter:
         self.feed_options = feed_options
         if feed_options is not None:
             self.item_classes = tuple(
-                load_object(item_class) for item_class in feed_options.get("item_classes") or ()
+                load_object(item_class)
+                for item_class in feed_options.get("item_classes") or ()
             )
         else:
             self.item_classes = tuple()
@@ -98,13 +100,12 @@ class IFeedStorage(Interface):
 
 @implementer(IFeedStorage)
 class BlockingFeedStorage:
-
     def open(self, spider):
-        path = spider.crawler.settings['FEED_TEMPDIR']
+        path = spider.crawler.settings["FEED_TEMPDIR"]
         if path and not Path(path).is_dir():
-            raise OSError('Not a Directory: ' + str(path))
+            raise OSError("Not a Directory: " + str(path))
 
-        return NamedTemporaryFile(prefix='feed-', dir=path)
+        return NamedTemporaryFile(prefix="feed-", dir=path)
 
     def store(self, file):
         return threads.deferToThread(self._store_in_thread, file)
@@ -115,16 +116,17 @@ class BlockingFeedStorage:
 
 @implementer(IFeedStorage)
 class StdoutFeedStorage:
-
     def __init__(self, uri, _stdout=None, *, feed_options=None):
         if not _stdout:
             _stdout = sys.stdout.buffer
         self._stdout = _stdout
-        if feed_options and feed_options.get('overwrite', False) is True:
-            logger.warning('Standard output (stdout) storage does not support '
-                           'overwriting. To suppress this warning, remove the '
-                           'overwrite option from your FEEDS setting, or set '
-                           'it to False.')
+        if feed_options and feed_options.get("overwrite", False) is True:
+            logger.warning(
+                "Standard output (stdout) storage does not support "
+                "overwriting. To suppress this warning, remove the "
+                "overwrite option from your FEEDS setting, or set "
+                "it to False."
+            )
 
     def open(self, spider):
         return self._stdout
@@ -135,11 +137,10 @@ class StdoutFeedStorage:
 
 @implementer(IFeedStorage)
 class FileFeedStorage:
-
     def __init__(self, uri, *, feed_options=None):
         self.path = file_uri_to_path(uri)
         feed_options = feed_options or {}
-        self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
+        self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab"
 
     def open(self, spider) -> IO[Any]:
         dirname = Path(self.path).parent
@@ -152,11 +153,19 @@ class FileFeedStorage:
 
 
 class S3FeedStorage(BlockingFeedStorage):
-
-    def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
-                 feed_options=None, session_token=None):
+    def __init__(
+        self,
+        uri,
+        access_key=None,
+        secret_key=None,
+        acl=None,
+        endpoint_url=None,
+        *,
+        feed_options=None,
+        session_token=None,
+    ):
         if not is_botocore_available():
-            raise NotConfigured('missing botocore library')
+            raise NotConfigured("missing botocore library")
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
@@ -166,41 +175,45 @@ class S3FeedStorage(BlockingFeedStorage):
         self.acl = acl
         self.endpoint_url = endpoint_url
         import botocore.session
+
         session = botocore.session.get_session()
         self.s3_client = session.create_client(
-            's3', aws_access_key_id=self.access_key,
+            "s3",
+            aws_access_key_id=self.access_key,
             aws_secret_access_key=self.secret_key,
             aws_session_token=self.session_token,
-            endpoint_url=self.endpoint_url)
-        if feed_options and feed_options.get('overwrite', True) is False:
-            logger.warning('S3 does not support appending to files. To '
-                           'suppress this warning, remove the overwrite '
-                           'option from your FEEDS setting or set it to True.')
+            endpoint_url=self.endpoint_url,
+        )
+        if feed_options and feed_options.get("overwrite", True) is False:
+            logger.warning(
+                "S3 does not support appending to files. To "
+                "suppress this warning, remove the overwrite "
+                "option from your FEEDS setting or set it to True."
+            )
 
     @classmethod
     def from_crawler(cls, crawler, uri, *, feed_options=None):
         return build_storage(
             cls,
             uri,
-            access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
-            secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
-            session_token=crawler.settings['AWS_SESSION_TOKEN'],
-            acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
-            endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
+            access_key=crawler.settings["AWS_ACCESS_KEY_ID"],
+            secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"],
+            session_token=crawler.settings["AWS_SESSION_TOKEN"],
+            acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
+            endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
             feed_options=feed_options,
         )
 
     def _store_in_thread(self, file):
         file.seek(0)
-        kwargs = {'ACL': self.acl} if self.acl else {}
+        kwargs = {"ACL": self.acl} if self.acl else {}
         self.s3_client.put_object(
-            Bucket=self.bucketname, Key=self.keyname, Body=file,
-            **kwargs)
+            Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
+        )
         file.close()
 
 
 class GCSFeedStorage(BlockingFeedStorage):
-
     def __init__(self, uri, project_id, acl):
         self.project_id = project_id
         self.acl = acl
@@ -212,13 +225,14 @@ class GCSFeedStorage(BlockingFeedStorage):
     def from_crawler(cls, crawler, uri):
         return cls(
             uri,
-            crawler.settings['GCS_PROJECT_ID'],
-            crawler.settings['FEED_STORAGE_GCS_ACL'] or None
+            crawler.settings["GCS_PROJECT_ID"],
+            crawler.settings["FEED_STORAGE_GCS_ACL"] or None,
         )
 
     def _store_in_thread(self, file):
         file.seek(0)
         from google.cloud.storage import Client
+
         client = Client(project=self.project_id)
         bucket = client.get_bucket(self.bucket_name)
         blob = bucket.blob(self.blob_name)
@@ -226,37 +240,51 @@ class GCSFeedStorage(BlockingFeedStorage):
 
 
 class FTPFeedStorage(BlockingFeedStorage):
-
     def __init__(self, uri, use_active_mode=False, *, feed_options=None):
         u = urlparse(uri)
         self.host = u.hostname
-        self.port = int(u.port or '21')
+        self.port = int(u.port or "21")
         self.username = u.username
-        self.password = unquote(u.password or '')
+        self.password = unquote(u.password or "")
         self.path = u.path
         self.use_active_mode = use_active_mode
-        self.overwrite = not feed_options or feed_options.get('overwrite', True)
+        self.overwrite = not feed_options or feed_options.get("overwrite", True)
 
     @classmethod
     def from_crawler(cls, crawler, uri, *, feed_options=None):
         return build_storage(
             cls,
             uri,
-            crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
+            crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"),
             feed_options=feed_options,
         )
 
     def _store_in_thread(self, file):
         ftp_store_file(
-            path=self.path, file=file, host=self.host,
-            port=self.port, username=self.username,
-            password=self.password, use_active_mode=self.use_active_mode,
+            path=self.path,
+            file=file,
+            host=self.host,
+            port=self.port,
+            username=self.username,
+            password=self.password,
+            use_active_mode=self.use_active_mode,
             overwrite=self.overwrite,
         )
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter):
+    def __init__(
+        self,
+        file,
+        exporter,
+        storage,
+        uri,
+        format,
+        store_empty,
+        batch_id,
+        uri_template,
+        filter,
+    ):
         self.file = file
         self.exporter = exporter
         self.storage = storage
@@ -283,7 +311,6 @@ class _FeedSlot:
 
 
 class FeedExporter:
-
     @classmethod
     def from_crawler(cls, crawler):
         exporter = cls(crawler)
@@ -299,48 +326,55 @@ class FeedExporter:
         self.slots = []
         self.filters = {}
 
-        if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
+        if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
             raise NotConfigured
 
         # Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings
-        if self.settings['FEED_URI']:
+        if self.settings["FEED_URI"]:
             warnings.warn(
-                'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of '
-                'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details',
-                category=ScrapyDeprecationWarning, stacklevel=2,
+                "The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of "
+                "the `FEEDS` setting. Please see the `FEEDS` setting docs for more details",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            uri = str(self.settings["FEED_URI"])  # handle pathlib.Path objects
+            feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
+            self.feeds[uri] = feed_complete_default_values_from_settings(
+                feed_options, self.settings
             )
-            uri = str(self.settings['FEED_URI'])  # handle pathlib.Path objects
-            feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
-            self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
             self.filters[uri] = self._load_filter(feed_options)
         # End: Backward compatibility for FEED_URI and FEED_FORMAT settings
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
-        for uri, feed_options in self.settings.getdict('FEEDS').items():
+        for uri, feed_options in self.settings.getdict("FEEDS").items():
             uri = str(uri)  # handle pathlib.Path objects
-            self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
+            self.feeds[uri] = feed_complete_default_values_from_settings(
+                feed_options, self.settings
+            )
             self.filters[uri] = self._load_filter(feed_options)
 
-        self.storages = self._load_components('FEED_STORAGES')
-        self.exporters = self._load_components('FEED_EXPORTERS')
+        self.storages = self._load_components("FEED_STORAGES")
+        self.exporters = self._load_components("FEED_EXPORTERS")
         for uri, feed_options in self.feeds.items():
             if not self._storage_supported(uri, feed_options):
                 raise NotConfigured
             if not self._settings_are_valid():
                 raise NotConfigured
-            if not self._exporter_supported(feed_options['format']):
+            if not self._exporter_supported(feed_options["format"]):
                 raise NotConfigured
 
     def open_spider(self, spider):
         for uri, feed_options in self.feeds.items():
-            uri_params = self._get_uri_params(spider, feed_options['uri_params'])
-            self.slots.append(self._start_new_batch(
-                batch_id=1,
-                uri=uri % uri_params,
-                feed_options=feed_options,
-                spider=spider,
-                uri_template=uri,
-            ))
+            uri_params = self._get_uri_params(spider, feed_options["uri_params"])
+            self.slots.append(
+                self._start_new_batch(
+                    batch_id=1,
+                    uri=uri % uri_params,
+                    feed_options=feed_options,
+                    spider=spider,
+                    uri_template=uri,
+                )
+            )
 
     def close_spider(self, spider):
         deferred_list = []
@@ -368,16 +402,15 @@ class FeedExporter:
 
     def _handle_store_error(self, f, logmsg, spider, slot_type):
         logger.error(
-            "Error storing %s", logmsg,
-            exc_info=failure_to_exc_info(f), extra={'spider': spider}
+            "Error storing %s",
+            logmsg,
+            exc_info=failure_to_exc_info(f),
+            extra={"spider": spider},
         )
         self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
 
     def _handle_store_success(self, f, logmsg, spider, slot_type):
-        logger.info(
-            "Stored %s", logmsg,
-            extra={'spider': spider}
-        )
+        logger.info("Stored %s", logmsg, extra={"spider": spider})
         self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
 
     def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
@@ -393,26 +426,28 @@ class FeedExporter:
         storage = self._get_storage(uri, feed_options)
         file = storage.open(spider)
         if "postprocessing" in feed_options:
-            file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)
+            file = PostProcessingManager(
+                feed_options["postprocessing"], file, feed_options
+            )
 
         exporter = self._get_exporter(
             file=file,
-            format=feed_options['format'],
-            fields_to_export=feed_options['fields'],
-            encoding=feed_options['encoding'],
-            indent=feed_options['indent'],
-            **feed_options['item_export_kwargs'],
+            format=feed_options["format"],
+            fields_to_export=feed_options["fields"],
+            encoding=feed_options["encoding"],
+            indent=feed_options["indent"],
+            **feed_options["item_export_kwargs"],
         )
         slot = _FeedSlot(
             file=file,
             exporter=exporter,
             storage=storage,
             uri=uri,
-            format=feed_options['format'],
-            store_empty=feed_options['store_empty'],
+            format=feed_options["format"],
+            store_empty=feed_options["store_empty"],
             batch_id=batch_id,
             uri_template=uri_template,
-            filter=self.filters[uri_template]
+            filter=self.filters[uri_template],
         )
         if slot.store_empty:
             slot.start_exporting()
@@ -422,7 +457,9 @@ class FeedExporter:
         slots = []
         for slot in self.slots:
             if not slot.filter.accepts(item):
-                slots.append(slot)    # if slot doesn't accept item, continue with next slot
+                slots.append(
+                    slot
+                )  # if slot doesn't accept item, continue with next slot
                 continue
 
             slot.start_exporting()
@@ -430,18 +467,22 @@ class FeedExporter:
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
             if (
-                self.feeds[slot.uri_template]['batch_item_count']
-                and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+                self.feeds[slot.uri_template]["batch_item_count"]
+                and slot.itemcount >= self.feeds[slot.uri_template]["batch_item_count"]
             ):
-                uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
+                uri_params = self._get_uri_params(
+                    spider, self.feeds[slot.uri_template]["uri_params"], slot
+                )
                 self._close_slot(slot, spider)
-                slots.append(self._start_new_batch(
-                    batch_id=slot.batch_id + 1,
-                    uri=slot.uri_template % uri_params,
-                    feed_options=self.feeds[slot.uri_template],
-                    spider=spider,
-                    uri_template=slot.uri_template,
-                ))
+                slots.append(
+                    self._start_new_batch(
+                        batch_id=slot.batch_id + 1,
+                        uri=slot.uri_template % uri_params,
+                        feed_options=self.feeds[slot.uri_template],
+                        spider=spider,
+                        uri_template=slot.uri_template,
+                    )
+                )
             else:
                 slots.append(slot)
         self.slots = slots
@@ -459,7 +500,7 @@ class FeedExporter:
     def _exporter_supported(self, format):
         if format in self.exporters:
             return True
-        logger.error("Unknown feed format: %(format)s", {'format': format})
+        logger.error("Unknown feed format: %(format)s", {"format": format})
 
     def _settings_are_valid(self):
         """
@@ -467,12 +508,14 @@ class FeedExporter:
         %(batch_time)s or %(batch_id)d to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
+            if values["batch_item_count"] and not re.search(
+                r"%\(batch_time\)s|%\(batch_id\)", uri_template
+            ):
                 logger.error(
-                    '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
-                    'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
-                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
-                    uri_template
+                    "%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT "
+                    "setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: "
+                    "https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count",
+                    uri_template,
                 )
                 return False
         return True
@@ -484,17 +527,17 @@ class FeedExporter:
                 self._get_storage(uri, feed_options)
                 return True
             except NotConfigured as e:
-                logger.error("Disabled feed storage scheme: %(scheme)s. "
-                             "Reason: %(reason)s",
-                             {'scheme': scheme, 'reason': str(e)})
+                logger.error(
+                    "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s",
+                    {"scheme": scheme, "reason": str(e)},
+                )
         else:
-            logger.error("Unknown feed storage scheme: %(scheme)s",
-                         {'scheme': scheme})
+            logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})
 
     def _get_instance(self, objcls, *args, **kwargs):
         return create_instance(
-            objcls, self.settings, getattr(self, 'crawler', None),
-            *args, **kwargs)
+            objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs
+        )
 
     def _get_exporter(self, file, format, *args, **kwargs):
         return self._get_instance(self.exporters[format], file, *args, **kwargs)
@@ -506,20 +549,22 @@ class FeedExporter:
         do not support it, and issuing a deprecation warning instead.
         """
         feedcls = self.storages[urlparse(uri).scheme]
-        crawler = getattr(self, 'crawler', None)
+        crawler = getattr(self, "crawler", None)
 
         def build_instance(builder, *preargs):
-            return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
+            return build_storage(
+                builder, uri, feed_options=feed_options, preargs=preargs
+            )
 
-        if crawler and hasattr(feedcls, 'from_crawler'):
+        if crawler and hasattr(feedcls, "from_crawler"):
             instance = build_instance(feedcls.from_crawler, crawler)
-            method_name = 'from_crawler'
-        elif hasattr(feedcls, 'from_settings'):
+            method_name = "from_crawler"
+        elif hasattr(feedcls, "from_settings"):
             instance = build_instance(feedcls.from_settings, self.settings)
-            method_name = 'from_settings'
+            method_name = "from_settings"
         else:
             instance = build_instance(feedcls)
-            method_name = '__new__'
+            method_name = "__new__"
         if instance is None:
             raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
         return instance
@@ -534,19 +579,23 @@ class FeedExporter:
         for k in dir(spider):
             params[k] = getattr(spider, k)
         utc_now = datetime.utcnow()
-        params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
-        params['batch_time'] = utc_now.isoformat().replace(':', '-')
-        params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
+        params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
+        params["batch_time"] = utc_now.isoformat().replace(":", "-")
+        params["batch_id"] = slot.batch_id + 1 if slot is not None else 1
         original_params = params.copy()
-        uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
+        uripar_function = (
+            load_object(uri_params_function)
+            if uri_params_function
+            else lambda params, _: params
+        )
         new_params = uripar_function(params, spider)
         if new_params is None or original_params != params:
             warnings.warn(
-                'Modifying the params dictionary in-place in the function defined in '
-                'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
-                'setting is deprecated. The function must return a new dictionary '
-                'instead.',
-                category=ScrapyDeprecationWarning
+                "Modifying the params dictionary in-place in the function defined in "
+                "the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS "
+                "setting is deprecated. The function must return a new dictionary "
+                "instead.",
+                category=ScrapyDeprecationWarning,
             )
         return new_params if new_params is not None else params
 
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 4e76fe5e5..3bfbddc54 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -21,10 +21,11 @@ logger = logging.getLogger(__name__)
 
 
 class DummyPolicy:
-
     def __init__(self, settings):
-        self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
-        self.ignore_http_codes = [int(x) for x in settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')]
+        self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
+        self.ignore_http_codes = [
+            int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES")
+        ]
 
     def should_cache_request(self, request):
         return urlparse_cached(request).scheme not in self.ignore_schemes
@@ -44,16 +45,17 @@ class RFC2616Policy:
     MAXAGE = 3600 * 24 * 365  # one year
 
     def __init__(self, settings):
-        self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
-        self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
+        self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE")
+        self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
         self._cc_parsed = WeakKeyDictionary()
         self.ignore_response_cache_controls = [
-            to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
+            to_bytes(cc)
+            for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS")
         ]
 
     def _parse_cachecontrol(self, r):
         if r not in self._cc_parsed:
-            cch = r.headers.get(b'Cache-Control', b'')
+            cch = r.headers.get(b"Cache-Control", b"")
             parsed = parse_cachecontrol(cch)
             if isinstance(r, Response):
                 for key in self.ignore_response_cache_controls:
@@ -66,7 +68,7 @@ class RFC2616Policy:
             return False
         cc = self._parse_cachecontrol(request)
         # obey user-agent directive "Cache-Control: no-store"
-        if b'no-store' in cc:
+        if b"no-store" in cc:
             return False
         # Any other is eligible for caching
         return True
@@ -77,7 +79,7 @@ class RFC2616Policy:
         # Status code 206 is not included because cache can not deal with partial contents
         cc = self._parse_cachecontrol(response)
         # obey directive "Cache-Control: no-store"
-        if b'no-store' in cc:
+        if b"no-store" in cc:
             return False
         # Never cache 304 (Not Modified) responses
         if response.status == 304:
@@ -86,14 +88,14 @@ class RFC2616Policy:
         if self.always_store:
             return True
         # Any hint on response expiration is good
-        if b'max-age' in cc or b'Expires' in response.headers:
+        if b"max-age" in cc or b"Expires" in response.headers:
             return True
         # Firefox fallbacks this statuses to one year expiration if none is set
         if response.status in (300, 301, 308):
             return True
         # Other statuses without expiration requires at least one validator
         if response.status in (200, 203, 401):
-            return b'Last-Modified' in response.headers or b'ETag' in response.headers
+            return b"Last-Modified" in response.headers or b"ETag" in response.headers
         # Any other is probably not eligible for caching
         # Makes no sense to cache responses that does not contain expiration
         # info and can not be revalidated
@@ -102,11 +104,13 @@ class RFC2616Policy:
     def is_cached_response_fresh(self, cachedresponse, request):
         cc = self._parse_cachecontrol(cachedresponse)
         ccreq = self._parse_cachecontrol(request)
-        if b'no-cache' in cc or b'no-cache' in ccreq:
+        if b"no-cache" in cc or b"no-cache" in ccreq:
             return False
 
         now = time()
-        freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now)
+        freshnesslifetime = self._compute_freshness_lifetime(
+            cachedresponse, request, now
+        )
         currentage = self._compute_current_age(cachedresponse, request, now)
 
         reqmaxage = self._get_max_age(ccreq)
@@ -116,7 +120,7 @@ class RFC2616Policy:
         if currentage < freshnesslifetime:
             return True
 
-        if b'max-stale' in ccreq and b'must-revalidate' not in cc:
+        if b"max-stale" in ccreq and b"must-revalidate" not in cc:
             # From RFC2616: "Indicates that the client is willing to
             # accept a response that has exceeded its expiration time.
             # If max-stale is assigned a value, then the client is
@@ -124,7 +128,7 @@ class RFC2616Policy:
             # expiration time by no more than the specified number of
             # seconds. If no value is assigned to max-stale, then the
             # client is willing to accept a stale response of any age."
-            staleage = ccreq[b'max-stale']
+            staleage = ccreq[b"max-stale"]
             if staleage is None:
                 return True
 
@@ -143,22 +147,24 @@ class RFC2616Policy:
         # as long as the old response didn't specify must-revalidate.
         if response.status >= 500:
             cc = self._parse_cachecontrol(cachedresponse)
-            if b'must-revalidate' not in cc:
+            if b"must-revalidate" not in cc:
                 return True
 
         # Use the cached response if the server says it hasn't changed.
         return response.status == 304
 
     def _set_conditional_validators(self, request, cachedresponse):
-        if b'Last-Modified' in cachedresponse.headers:
-            request.headers[b'If-Modified-Since'] = cachedresponse.headers[b'Last-Modified']
+        if b"Last-Modified" in cachedresponse.headers:
+            request.headers[b"If-Modified-Since"] = cachedresponse.headers[
+                b"Last-Modified"
+            ]
 
-        if b'ETag' in cachedresponse.headers:
-            request.headers[b'If-None-Match'] = cachedresponse.headers[b'ETag']
+        if b"ETag" in cachedresponse.headers:
+            request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"]
 
     def _get_max_age(self, cc):
         try:
-            return max(0, int(cc[b'max-age']))
+            return max(0, int(cc[b"max-age"]))
         except (KeyError, ValueError):
             return None
 
@@ -171,18 +177,18 @@ class RFC2616Policy:
             return maxage
 
         # Parse date header or synthesize it if none exists
-        date = rfc1123_to_epoch(response.headers.get(b'Date')) or now
+        date = rfc1123_to_epoch(response.headers.get(b"Date")) or now
 
         # Try HTTP/1.0 Expires header
-        if b'Expires' in response.headers:
-            expires = rfc1123_to_epoch(response.headers[b'Expires'])
+        if b"Expires" in response.headers:
+            expires = rfc1123_to_epoch(response.headers[b"Expires"])
             # When parsing Expires header fails RFC 2616 section 14.21 says we
             # should treat this as an expiration time in the past.
             return max(0, expires - date) if expires else 0
 
         # Fallback to heuristic using last-modified header
         # This is not in RFC but on Firefox caching implementation
-        lastmodified = rfc1123_to_epoch(response.headers.get(b'Last-Modified'))
+        lastmodified = rfc1123_to_epoch(response.headers.get(b"Last-Modified"))
         if lastmodified and lastmodified <= date:
             return (date - lastmodified) / 10
 
@@ -199,13 +205,13 @@ class RFC2616Policy:
         currentage = 0
         # If Date header is not set we assume it is a fast connection, and
         # clock is in sync with the server
-        date = rfc1123_to_epoch(response.headers.get(b'Date')) or now
+        date = rfc1123_to_epoch(response.headers.get(b"Date")) or now
         if now > date:
             currentage = now - date
 
-        if b'Age' in response.headers:
+        if b"Age" in response.headers:
             try:
-                age = int(response.headers[b'Age'])
+                age = int(response.headers[b"Age"])
                 currentage = max(currentage, age)
             except ValueError:
                 pass
@@ -214,18 +220,21 @@ class RFC2616Policy:
 
 
 class DbmCacheStorage:
-
     def __init__(self, settings):
-        self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
-        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
+        self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True)
+        self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
+        self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"])
         self.db = None
 
     def open_spider(self, spider: Spider):
-        dbpath = Path(self.cachedir, f'{spider.name}.db')
-        self.db = self.dbmodule.open(str(dbpath), 'c')
+        dbpath = Path(self.cachedir, f"{spider.name}.db")
+        self.db = self.dbmodule.open(str(dbpath), "c")
 
-        logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
+        logger.debug(
+            "Using DBM cache storage in %(cachepath)s",
+            {"cachepath": dbpath},
+            extra={"spider": spider},
+        )
 
         self._fingerprinter = spider.crawler.request_fingerprinter
 
@@ -236,10 +245,10 @@ class DbmCacheStorage:
         data = self._read_data(spider, request)
         if data is None:
             return  # not cached
-        url = data['url']
-        status = data['status']
-        headers = Headers(data['headers'])
-        body = data['body']
+        url = data["url"]
+        status = data["status"]
+        headers = Headers(data["headers"])
+        body = data["body"]
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
@@ -247,18 +256,18 @@ class DbmCacheStorage:
     def store_response(self, spider, request, response):
         key = self._fingerprinter.fingerprint(request).hex()
         data = {
-            'status': response.status,
-            'url': response.url,
-            'headers': dict(response.headers),
-            'body': response.body,
+            "status": response.status,
+            "url": response.url,
+            "headers": dict(response.headers),
+            "body": response.body,
         }
-        self.db[f'{key}_data'] = pickle.dumps(data, protocol=4)
-        self.db[f'{key}_time'] = str(time())
+        self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
+        self.db[f"{key}_time"] = str(time())
 
     def _read_data(self, spider, request):
         key = self._fingerprinter.fingerprint(request).hex()
         db = self.db
-        tkey = f'{key}_time'
+        tkey = f"{key}_time"
         if tkey not in db:
             return  # not found
 
@@ -266,20 +275,22 @@ class DbmCacheStorage:
         if 0 < self.expiration_secs < time() - float(ts):
             return  # expired
 
-        return pickle.loads(db[f'{key}_data'])
+        return pickle.loads(db[f"{key}_data"])
 
 
 class FilesystemCacheStorage:
-
     def __init__(self, settings):
-        self.cachedir = data_path(settings['HTTPCACHE_DIR'])
-        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
+        self.cachedir = data_path(settings["HTTPCACHE_DIR"])
+        self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
+        self.use_gzip = settings.getbool("HTTPCACHE_GZIP")
         self._open = gzip.open if self.use_gzip else open
 
     def open_spider(self, spider: Spider):
-        logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
-                     extra={'spider': spider})
+        logger.debug(
+            "Using filesystem cache storage in %(cachedir)s",
+            {"cachedir": self.cachedir},
+            extra={"spider": spider},
+        )
 
         self._fingerprinter = spider.crawler.request_fingerprinter
 
@@ -292,12 +303,12 @@ class FilesystemCacheStorage:
         if metadata is None:
             return  # not cached
         rpath = Path(self._get_request_path(spider, request))
-        with self._open(rpath / 'response_body', 'rb') as f:
+        with self._open(rpath / "response_body", "rb") as f:
             body = f.read()
-        with self._open(rpath / 'response_headers', 'rb') as f:
+        with self._open(rpath / "response_headers", "rb") as f:
             rawheaders = f.read()
-        url = metadata.get('response_url')
-        status = metadata['status']
+        url = metadata.get("response_url")
+        status = metadata["status"]
         headers = Headers(headers_raw_to_dict(rawheaders))
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
@@ -309,23 +320,23 @@ class FilesystemCacheStorage:
         if not rpath.exists():
             rpath.mkdir(parents=True)
         metadata = {
-            'url': request.url,
-            'method': request.method,
-            'status': response.status,
-            'response_url': response.url,
-            'timestamp': time(),
+            "url": request.url,
+            "method": request.method,
+            "status": response.status,
+            "response_url": response.url,
+            "timestamp": time(),
         }
-        with self._open(rpath / 'meta', 'wb') as f:
+        with self._open(rpath / "meta", "wb") as f:
             f.write(to_bytes(repr(metadata)))
-        with self._open(rpath / 'pickled_meta', 'wb') as f:
+        with self._open(rpath / "pickled_meta", "wb") as f:
             pickle.dump(metadata, f, protocol=4)
-        with self._open(rpath / 'response_headers', 'wb') as f:
+        with self._open(rpath / "response_headers", "wb") as f:
             f.write(headers_dict_to_raw(response.headers))
-        with self._open(rpath / 'response_body', 'wb') as f:
+        with self._open(rpath / "response_body", "wb") as f:
             f.write(response.body)
-        with self._open(rpath / 'request_headers', 'wb') as f:
+        with self._open(rpath / "request_headers", "wb") as f:
             f.write(headers_dict_to_raw(request.headers))
-        with self._open(rpath / 'request_body', 'wb') as f:
+        with self._open(rpath / "request_body", "wb") as f:
             f.write(request.body)
 
     def _get_request_path(self, spider: Spider, request: Request) -> str:
@@ -334,13 +345,13 @@ class FilesystemCacheStorage:
 
     def _read_meta(self, spider: Spider, request: Request):
         rpath = Path(self._get_request_path(spider, request))
-        metapath = rpath / 'pickled_meta'
+        metapath = rpath / "pickled_meta"
         if not metapath.exists():
             return  # not found
         mtime = metapath.stat().st_mtime
         if 0 < self.expiration_secs < time() - mtime:
             return  # expired
-        with self._open(metapath, 'rb') as f:
+        with self._open(metapath, "rb") as f:
             return pickle.load(f)
 
 
@@ -357,8 +368,8 @@ def parse_cachecontrol(header):
 
     """
     directives = {}
-    for directive in header.split(b','):
-        key, sep, val = directive.strip().partition(b'=')
+    for directive in header.split(b","):
+        key, sep, val = directive.strip().partition(b"=")
         if key:
             directives[key.lower()] = val if sep else None
     return directives
@@ -366,7 +377,7 @@ def parse_cachecontrol(header):
 
 def rfc1123_to_epoch(date_str):
     try:
-        date_str = to_unicode(date_str, encoding='ascii')
+        date_str = to_unicode(date_str, encoding="ascii")
         return mktime_tz(parsedate_tz(date_str))
     except Exception:
         return None
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 0be2831a1..6295dcdb7 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -19,7 +19,7 @@ class LogStats:
 
     @classmethod
     def from_crawler(cls, crawler):
-        interval = crawler.settings.getfloat('LOGSTATS_INTERVAL')
+        interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
         if not interval:
             raise NotConfigured
         o = cls(crawler.stats, interval)
@@ -35,17 +35,23 @@ class LogStats:
         self.task.start(self.interval)
 
     def log(self, spider):
-        items = self.stats.get_value('item_scraped_count', 0)
-        pages = self.stats.get_value('response_received_count', 0)
+        items = self.stats.get_value("item_scraped_count", 0)
+        pages = self.stats.get_value("response_received_count", 0)
         irate = (items - self.itemsprev) * self.multiplier
         prate = (pages - self.pagesprev) * self.multiplier
         self.pagesprev, self.itemsprev = pages, items
 
-        msg = ("Crawled %(pages)d pages (at %(pagerate)d pages/min), "
-               "scraped %(items)d items (at %(itemrate)d items/min)")
-        log_args = {'pages': pages, 'pagerate': prate,
-                    'items': items, 'itemrate': irate}
-        logger.info(msg, log_args, extra={'spider': spider})
+        msg = (
+            "Crawled %(pages)d pages (at %(pagerate)d pages/min), "
+            "scraped %(items)d items (at %(itemrate)d items/min)"
+        )
+        log_args = {
+            "pages": pages,
+            "pagerate": prate,
+            "items": items,
+            "itemrate": irate,
+        }
+        logger.info(msg, log_args, extra={"spider": spider})
 
     def spider_closed(self, spider, reason):
         if self.task and self.task.running:
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index cee44ea62..03ede0681 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -12,13 +12,12 @@ from scrapy.utils.trackref import live_refs
 
 
 class MemoryDebugger:
-
     def __init__(self, stats):
         self.stats = stats
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
+        if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
             raise NotConfigured
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@@ -26,8 +25,12 @@ class MemoryDebugger:
 
     def spider_closed(self, spider, reason):
         gc.collect()
-        self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
+        self.stats.set_value(
+            "memdebug/gc_garbage_count", len(gc.garbage), spider=spider
+        )
         for cls, wdict in live_refs.items():
             if not wdict:
                 continue
-            self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)
+            self.stats.set_value(
+                f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider
+            )
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 4fdf86479..2bba71972 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -20,22 +20,23 @@ logger = logging.getLogger(__name__)
 
 
 class MemoryUsage:
-
     def __init__(self, crawler):
-        if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
+        if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
             raise NotConfigured
         try:
             # stdlib's resource module is only available on unix platforms.
-            self.resource = import_module('resource')
+            self.resource = import_module("resource")
         except ImportError:
             raise NotConfigured
 
         self.crawler = crawler
         self.warned = False
-        self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
-        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
-        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
-        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
+        self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
+        self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
+        self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
+        self.check_interval = crawler.settings.getfloat(
+            "MEMUSAGE_CHECK_INTERVAL_SECONDS"
+        )
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
@@ -46,13 +47,13 @@ class MemoryUsage:
 
     def get_virtual_size(self):
         size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
-        if sys.platform != 'darwin':
+        if sys.platform != "darwin":
             # on macOS ru_maxrss is in bytes, on Linux it is in KB
             size *= 1024
         return size
 
     def engine_started(self):
-        self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())
+        self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
         self.tasks = []
         tsk = task.LoopingCall(self.update)
         self.tasks.append(tsk)
@@ -72,45 +73,56 @@ class MemoryUsage:
                 tsk.stop()
 
     def update(self):
-        self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
+        self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
 
     def _check_limit(self):
         peak_mem_usage = self.get_virtual_size()
         if peak_mem_usage > self.limit:
-            self.crawler.stats.set_value('memusage/limit_reached', 1)
+            self.crawler.stats.set_value("memusage/limit_reached", 1)
             mem = self.limit / 1024 / 1024
-            logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
-                         {'memusage': mem}, extra={'crawler': self.crawler})
+            logger.error(
+                "Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
+                {"memusage": mem},
+                extra={"crawler": self.crawler},
+            )
             if self.notify_mails:
                 subj = (
                     f"{self.crawler.settings['BOT_NAME']} terminated: "
                     f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
                 )
                 self._send_report(self.notify_mails, subj)
-                self.crawler.stats.set_value('memusage/limit_notified', 1)
+                self.crawler.stats.set_value("memusage/limit_notified", 1)
 
             if self.crawler.engine.spider is not None:
-                self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
+                self.crawler.engine.close_spider(
+                    self.crawler.engine.spider, "memusage_exceeded"
+                )
             else:
                 self.crawler.stop()
         else:
-            logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024})
+            logger.info(
+                "Peak memory usage is %(virtualsize)dMiB",
+                {"virtualsize": peak_mem_usage / 1024 / 1024},
+            )
 
     def _check_warning(self):
         if self.warned:  # warn only once
             return
         if self.get_virtual_size() > self.warning:
-            self.crawler.stats.set_value('memusage/warning_reached', 1)
+            self.crawler.stats.set_value("memusage/warning_reached", 1)
             mem = self.warning / 1024 / 1024
-            logger.warning("Memory usage reached %(memusage)dMiB",
-                           {'memusage': mem}, extra={'crawler': self.crawler})
+            logger.warning(
+                "Memory usage reached %(memusage)dMiB",
+                {"memusage": mem},
+                extra={"crawler": self.crawler},
+            )
             if self.notify_mails:
                 subj = (
                     f"{self.crawler.settings['BOT_NAME']} warning: "
                     f"memory usage reached {mem}MiB at {socket.gethostname()}"
                 )
                 self._send_report(self.notify_mails, subj)
-                self.crawler.stats.set_value('memusage/warning_notified', 1)
+                self.crawler.stats.set_value("memusage/warning_notified", 1)
             self.warned = True
 
     def _send_report(self, rcpts, subject):
@@ -120,7 +132,9 @@ class MemoryUsage:
         s += f"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\r\n"
         s += f"Current memory usage          : {self.get_virtual_size()/1024/1024}M\r\n"
 
-        s += "ENGINE STATUS ------------------------------------------------------- \r\n"
+        s += (
+            "ENGINE STATUS ------------------------------------------------------- \r\n"
+        )
         s += "\r\n"
         s += pformat(get_engine_status(self.crawler.engine))
         s += "\r\n"
diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py
index 413c2e55e..79e3b1656 100644
--- a/scrapy/extensions/postprocessing.py
+++ b/scrapy/extensions/postprocessing.py
@@ -29,8 +29,13 @@ class GzipPlugin:
         compress_level = self.feed_options.get("gzip_compresslevel", 9)
         mtime = self.feed_options.get("gzip_mtime")
         filename = self.feed_options.get("gzip_filename")
-        self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,
-                                 mtime=mtime, filename=filename)
+        self.gzipfile = GzipFile(
+            fileobj=self.file,
+            mode="wb",
+            compresslevel=compress_level,
+            mtime=mtime,
+            filename=filename,
+        )
 
     def write(self, data: bytes) -> int:
         return self.gzipfile.write(data)
@@ -55,7 +60,9 @@ class Bz2Plugin:
         self.file = file
         self.feed_options = feed_options
         compress_level = self.feed_options.get("bz2_compresslevel", 9)
-        self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)
+        self.bz2file = BZ2File(
+            filename=self.file, mode="wb", compresslevel=compress_level
+        )
 
     def write(self, data: bytes) -> int:
         return self.bz2file.write(data)
@@ -90,8 +97,14 @@ class LZMAPlugin:
         check = self.feed_options.get("lzma_check", -1)
         preset = self.feed_options.get("lzma_preset")
         filters = self.feed_options.get("lzma_filters")
-        self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,
-                                 check=check, preset=preset, filters=filters)
+        self.lzmafile = LZMAFile(
+            filename=self.file,
+            mode="wb",
+            format=format,
+            check=check,
+            preset=preset,
+            filters=filters,
+        )
 
     def write(self, data: bytes) -> int:
         return self.lzmafile.write(data)
@@ -114,7 +127,9 @@ class PostProcessingManager(IOBase):
     :type file: file like object
     """
 
-    def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+    def __init__(
+        self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]
+    ) -> None:
         self.plugins = self._load_plugins(plugins)
         self.file = file
         self.feed_options = feed_options
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index e9c8b1d6a..929a3be70 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -25,16 +25,16 @@ class SpiderState:
 
     def spider_closed(self, spider):
         if self.jobdir:
-            with Path(self.statefn).open('wb') as f:
+            with Path(self.statefn).open("wb") as f:
                 pickle.dump(spider.state, f, protocol=4)
 
     def spider_opened(self, spider):
         if self.jobdir and Path(self.statefn).exists():
-            with Path(self.statefn).open('rb') as f:
+            with Path(self.statefn).open("rb") as f:
                 spider.state = pickle.load(f)
         else:
             spider.state = {}
 
     @property
     def statefn(self) -> str:
-        return str(Path(self.jobdir, 'spider.state'))
+        return str(Path(self.jobdir, "spider.state"))
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 739e6b958..8733ad22b 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -10,7 +10,6 @@ from scrapy.exceptions import NotConfigured
 
 
 class StatsMailer:
-
     def __init__(self, stats, recipients, mail):
         self.stats = stats
         self.recipients = recipients
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 054350e4a..271f22428 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -15,6 +15,7 @@ from twisted.internet import protocol
 try:
     from twisted.conch import manhole, telnet
     from twisted.conch.insults import insults
+
     TWISTED_CONCH_AVAILABLE = True
 except (ImportError, SyntaxError):
     _TWISTED_CONCH_TRACEBACK = traceback.format_exc()
@@ -35,24 +36,26 @@ update_telnet_vars = object()
 
 
 class TelnetConsole(protocol.ServerFactory):
-
     def __init__(self, crawler):
-        if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
+        if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
             raise NotConfigured
         if not TWISTED_CONCH_AVAILABLE:
             raise NotConfigured(
-                'TELNETCONSOLE_ENABLED setting is True but required twisted '
-                'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK)
+                "TELNETCONSOLE_ENABLED setting is True but required twisted "
+                "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK
+            )
         self.crawler = crawler
         self.noisy = False
-        self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
-        self.host = crawler.settings['TELNETCONSOLE_HOST']
-        self.username = crawler.settings['TELNETCONSOLE_USERNAME']
-        self.password = crawler.settings['TELNETCONSOLE_PASSWORD']
+        self.portrange = [
+            int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT")
+        ]
+        self.host = crawler.settings["TELNETCONSOLE_HOST"]
+        self.username = crawler.settings["TELNETCONSOLE_USERNAME"]
+        self.password = crawler.settings["TELNETCONSOLE_PASSWORD"]
 
         if not self.password:
-            self.password = binascii.hexlify(os.urandom(8)).decode('utf8')
-            logger.info('Telnet Password: %s', self.password)
+            self.password = binascii.hexlify(os.urandom(8)).decode("utf8")
+            logger.info("Telnet Password: %s", self.password)
 
         self.crawler.signals.connect(self.start_listening, signals.engine_started)
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
@@ -64,9 +67,11 @@ class TelnetConsole(protocol.ServerFactory):
     def start_listening(self):
         self.port = listen_tcp(self.portrange, self.host, self)
         h = self.port.getHost()
-        logger.info("Telnet console listening on %(host)s:%(port)d",
-                    {'host': h.host, 'port': h.port},
-                    extra={'crawler': self.crawler})
+        logger.info(
+            "Telnet console listening on %(host)s:%(port)d",
+            {"host": h.host, "port": h.port},
+            extra={"crawler": self.crawler},
+        )
 
     def stop_listening(self):
         self.port.stopListening()
@@ -74,41 +79,37 @@ class TelnetConsole(protocol.ServerFactory):
     def protocol(self):
         class Portal:
             """An implementation of IPortal"""
+
             @defers
             def login(self_, credentials, mind, *interfaces):
                 if not (
-                    credentials.username == self.username.encode('utf8')
-                    and credentials.checkPassword(self.password.encode('utf8'))
+                    credentials.username == self.username.encode("utf8")
+                    and credentials.checkPassword(self.password.encode("utf8"))
                 ):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(
-                    insults.ServerProtocol,
-                    manhole.Manhole,
-                    self._get_telnet_vars()
+                    insults.ServerProtocol, manhole.Manhole, self._get_telnet_vars()
                 )
                 return (interfaces[0], protocol, lambda: None)
 
-        return telnet.TelnetTransport(
-            telnet.AuthenticatingTelnetProtocol,
-            Portal()
-        )
+        return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal())
 
     def _get_telnet_vars(self):
         # Note: if you add entries here also update topics/telnetconsole.rst
         telnet_vars = {
-            'engine': self.crawler.engine,
-            'spider': self.crawler.engine.spider,
-            'slot': self.crawler.engine.slot,
-            'crawler': self.crawler,
-            'extensions': self.crawler.extensions,
-            'stats': self.crawler.stats,
-            'settings': self.crawler.settings,
-            'est': lambda: print_engine_status(self.crawler.engine),
-            'p': pprint.pprint,
-            'prefs': print_live_refs,
-            'help': "This is Scrapy telnet console. For more info see: "
-                    "https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
+            "engine": self.crawler.engine,
+            "spider": self.crawler.engine.spider,
+            "slot": self.crawler.engine.slot,
+            "crawler": self.crawler,
+            "extensions": self.crawler.extensions,
+            "stats": self.crawler.stats,
+            "settings": self.crawler.settings,
+            "est": lambda: print_engine_status(self.crawler.engine),
+            "p": pprint.pprint,
+            "prefs": print_live_refs,
+            "help": "This is Scrapy telnet console. For more info see: "
+            "https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
         }
         self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
         return telnet_vars
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index eb21e426e..79e20de2a 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -7,16 +7,19 @@ logger = logging.getLogger(__name__)
 
 
 class AutoThrottle:
-
     def __init__(self, crawler):
         self.crawler = crawler
-        if not crawler.settings.getbool('AUTOTHROTTLE_ENABLED'):
+        if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
             raise NotConfigured
 
         self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
-        self.target_concurrency = crawler.settings.getfloat("AUTOTHROTTLE_TARGET_CONCURRENCY")
+        self.target_concurrency = crawler.settings.getfloat(
+            "AUTOTHROTTLE_TARGET_CONCURRENCY"
+        )
         crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
-        crawler.signals.connect(self._response_downloaded, signal=signals.response_downloaded)
+        crawler.signals.connect(
+            self._response_downloaded, signal=signals.response_downloaded
+        )
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -29,17 +32,19 @@ class AutoThrottle:
 
     def _min_delay(self, spider):
         s = self.crawler.settings
-        return getattr(spider, 'download_delay', s.getfloat('DOWNLOAD_DELAY'))
+        return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY"))
 
     def _max_delay(self, spider):
-        return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY')
+        return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
 
     def _start_delay(self, spider):
-        return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY'))
+        return max(
+            self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
+        )
 
     def _response_downloaded(self, response, request, spider):
         key, slot = self._get_slot(request, spider)
-        latency = request.meta.get('download_latency')
+        latency = request.meta.get("download_latency")
         if latency is None or slot is None:
             return
 
@@ -54,15 +59,18 @@ class AutoThrottle:
                 "delay:%(delay)5d ms (%(delaydiff)+d) | "
                 "latency:%(latency)5d ms | size:%(size)6d bytes",
                 {
-                    'slot': key, 'concurrency': conc,
-                    'delay': slot.delay * 1000, 'delaydiff': diff * 1000,
-                    'latency': latency * 1000, 'size': size,
+                    "slot": key,
+                    "concurrency": conc,
+                    "delay": slot.delay * 1000,
+                    "delaydiff": diff * 1000,
+                    "latency": latency * 1000,
+                    "size": size,
                 },
-                extra={'spider': spider}
+                extra={"spider": spider},
             )
 
     def _get_slot(self, request, spider):
-        key = request.meta.get('download_slot')
+        key = request.meta.get("download_slot")
         return key, self.crawler.engine.downloader.slots.get(key)
 
     def _adjust_delay(self, slot, latency, response):
diff --git a/scrapy/http/common.py b/scrapy/http/common.py
index 98699d7fd..bc8861574 100644
--- a/scrapy/http/common.py
+++ b/scrapy/http/common.py
@@ -3,4 +3,5 @@ def obsolete_setter(setter, attrname):
         c = self.__class__.__name__
         msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
         raise AttributeError(msg)
+
     return newsetter
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index b43c383fe..94afedb08 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -36,7 +36,7 @@ class CookieJar:
 
         if not IPV4_RE.search(req_host):
             hosts = potential_domain_matches(req_host)
-            if '.' not in req_host:
+            if "." not in req_host:
                 hosts += [req_host + ".local"]
         else:
             hosts = [req_host]
@@ -96,14 +96,14 @@ def potential_domain_matches(domain):
     """
     matches = [domain]
     try:
-        start = domain.index('.') + 1
-        end = domain.rindex('.')
+        start = domain.index(".") + 1
+        end = domain.rindex(".")
         while start < end:
             matches.append(domain[start:])
-            start = domain.index('.', start) + 1
+            start = domain.index(".", start) + 1
     except ValueError:
         pass
-    return matches + ['.' + d for d in matches]
+    return matches + ["." + d for d in matches]
 
 
 class _DummyLock:
@@ -140,7 +140,7 @@ class WrappedRequest:
         HTML document, and the user had no option to approve the automatic
         fetching of the image, this should be true.
         """
-        return self.request.meta.get('is_unverifiable', False)
+        return self.request.meta.get("is_unverifiable", False)
 
     @property
     def full_url(self):
@@ -166,13 +166,14 @@ class WrappedRequest:
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_unicode(self.request.headers.get(name, default),
-                          errors='replace')
+        return to_unicode(self.request.headers.get(name, default), errors="replace")
 
     def header_items(self):
         return [
-            (to_unicode(k, errors='replace'),
-             [to_unicode(x, errors='replace') for x in v])
+            (
+                to_unicode(k, errors="replace"),
+                [to_unicode(x, errors="replace") for x in v],
+            )
             for k, v in self.request.headers.items()
         ]
 
@@ -181,7 +182,6 @@ class WrappedRequest:
 
 
 class WrappedResponse:
-
     def __init__(self, response):
         self.response = response
 
@@ -189,5 +189,6 @@ class WrappedResponse:
         return self
 
     def get_all(self, name, default=None):
-        return [to_unicode(v, errors='replace')
-                for v in self.response.headers.getlist(name)]
+        return [
+            to_unicode(v, errors="replace") for v in self.response.headers.getlist(name)
+        ]
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 2e0020890..a5db30d6f 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -8,7 +8,7 @@ from scrapy.utils.python import to_unicode
 class Headers(CaselessDict):
     """Case insensitive http headers dictionary"""
 
-    def __init__(self, seq=None, encoding='utf-8'):
+    def __init__(self, seq=None, encoding="utf-8"):
         self.encoding = encoding
         super().__init__(seq)
 
@@ -29,7 +29,7 @@ class Headers(CaselessDict):
             value = []
         elif isinstance(value, (str, bytes)):
             value = [value]
-        elif not hasattr(value, '__iter__'):
+        elif not hasattr(value, "__iter__"):
             value = [value]
 
         return [self._tobytes(x) for x in value]
@@ -41,7 +41,7 @@ class Headers(CaselessDict):
             return x.encode(self.encoding)
         if isinstance(x, int):
             return str(x).encode(self.encoding)
-        raise TypeError(f'Unsupported value type: {type(x)}')
+        raise TypeError(f"Unsupported value type: {type(x)}")
 
     def __getitem__(self, key):
         try:
@@ -84,13 +84,16 @@ class Headers(CaselessDict):
         return headers_dict_to_raw(self)
 
     def to_unicode_dict(self):
-        """ Return headers as a CaselessDict with unicode keys
+        """Return headers as a CaselessDict with unicode keys
         and unicode values. Multiple values are joined with ','.
         """
         return CaselessDict(
-            (to_unicode(key, encoding=self.encoding),
-             to_unicode(b','.join(value), encoding=self.encoding))
-            for key, value in self.items())
+            (
+                to_unicode(key, encoding=self.encoding),
+                to_unicode(b",".join(value), encoding=self.encoding),
+            )
+            for key, value in self.items()
+        )
 
     def __copy__(self):
         return self.__class__(self)
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index a1001fc4a..e290f2143 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -27,9 +27,19 @@ class Request(object_ref):
     """
 
     attributes: Tuple[str, ...] = (
-        "url", "callback", "method", "headers", "body",
-        "cookies", "meta", "encoding", "priority",
-        "dont_filter", "errback", "flags", "cb_kwargs",
+        "url",
+        "callback",
+        "method",
+        "headers",
+        "body",
+        "cookies",
+        "meta",
+        "encoding",
+        "priority",
+        "dont_filter",
+        "errback",
+        "flags",
+        "cb_kwargs",
     )
     """A tuple of :class:`str` objects containing the name of all public
     attributes of the class that are also keyword parameters of the
@@ -64,9 +74,11 @@ class Request(object_ref):
         self.priority = priority
 
         if callback is not None and not callable(callback):
-            raise TypeError(f'callback must be a callable, got {type(callback).__name__}')
+            raise TypeError(
+                f"callback must be a callable, got {type(callback).__name__}"
+            )
         if errback is not None and not callable(errback):
-            raise TypeError(f'errback must be a callable, got {type(errback).__name__}')
+            raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
         self.callback = callback
         self.errback = errback
 
@@ -101,13 +113,13 @@ class Request(object_ref):
         self._url = escape_ajax(s)
 
         if (
-            '://' not in self._url
-            and not self._url.startswith('about:')
-            and not self._url.startswith('data:')
+            "://" not in self._url
+            and not self._url.startswith("about:")
+            and not self._url.startswith("data:")
         ):
-            raise ValueError(f'Missing scheme in request url: {self._url}')
+            raise ValueError(f"Missing scheme in request url: {self._url}")
 
-    url = property(_get_url, obsolete_setter(_set_url, 'url'))
+    url = property(_get_url, obsolete_setter(_set_url, "url"))
 
     def _get_body(self) -> bytes:
         return self._body
@@ -115,7 +127,7 @@ class Request(object_ref):
     def _set_body(self, body: Optional[Union[str, bytes]]) -> None:
         self._body = b"" if body is None else to_bytes(body, self.encoding)
 
-    body = property(_get_body, obsolete_setter(_set_body, 'body'))
+    body = property(_get_body, obsolete_setter(_set_body, "body"))
 
     @property
     def encoding(self) -> str:
@@ -131,12 +143,15 @@ class Request(object_ref):
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop('cls', self.__class__)
+        cls = kwargs.pop("cls", self.__class__)
         return cls(*args, **kwargs)
 
     @classmethod
     def from_curl(
-        cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs
+        cls: Type[RequestTypeVar],
+        curl_command: str,
+        ignore_unknown_options: bool = True,
+        **kwargs,
     ) -> RequestTypeVar:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
@@ -179,21 +194,25 @@ class Request(object_ref):
         """
         d = {
             "url": self.url,  # urls are safe (safe_string_url)
-            "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,
-            "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,
+            "callback": _find_method(spider, self.callback)
+            if callable(self.callback)
+            else self.callback,
+            "errback": _find_method(spider, self.errback)
+            if callable(self.errback)
+            else self.errback,
             "headers": dict(self.headers),
         }
         for attr in self.attributes:
             d.setdefault(attr, getattr(self, attr))
         if type(self) is not Request:  # pylint: disable=unidiomatic-typecheck
-            d["_class"] = self.__module__ + '.' + self.__class__.__name__
+            d["_class"] = self.__module__ + "." + self.__class__.__name__
         return d
 
 
 def _find_method(obj, func):
     """Helper function for Request.to_dict"""
     # Only instance methods contain ``__func__``
-    if obj and hasattr(func, '__func__'):
+    if obj and hasattr(func, "__func__"):
         members = inspect.getmembers(obj, predicate=inspect.ismethod)
         for name, obj_func in members:
             # We need to use __func__ to access the original function object because instance
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 8e0a7fae2..ea519fb19 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -24,22 +24,26 @@ FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
 
 
 class FormRequest(Request):
-    valid_form_methods = ['GET', 'POST']
+    valid_form_methods = ["GET", "POST"]
 
     def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
-        if formdata and kwargs.get('method') is None:
-            kwargs['method'] = 'POST'
+        if formdata and kwargs.get("method") is None:
+            kwargs["method"] = "POST"
 
         super().__init__(*args, **kwargs)
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
             form_query_str = _urlencode(items, self.encoding)
-            if self.method == 'POST':
-                self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
+            if self.method == "POST":
+                self.headers.setdefault(
+                    b"Content-Type", b"application/x-www-form-urlencoded"
+                )
                 self._set_body(form_query_str)
             else:
-                self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str)))
+                self._set_url(
+                    urlunsplit(urlsplit(self.url)._replace(query=form_query_str))
+                )
 
     @classmethod
     def from_response(
@@ -55,28 +59,29 @@ class FormRequest(Request):
         formcss: Optional[str] = None,
         **kwargs,
     ) -> FormRequestTypeVar:
-        kwargs.setdefault('encoding', response.encoding)
+        kwargs.setdefault("encoding", response.encoding)
 
         if formcss is not None:
             from parsel.csstranslator import HTMLTranslator
+
             formxpath = HTMLTranslator().css_to_xpath(formcss)
 
         form = _get_form(response, formname, formid, formnumber, formxpath)
         formdata = _get_inputs(form, formdata, dont_click, clickdata)
-        url = _get_form_url(form, kwargs.pop('url', None))
+        url = _get_form_url(form, kwargs.pop("url", None))
 
-        method = kwargs.pop('method', form.method)
+        method = kwargs.pop("method", form.method)
         if method is not None:
             method = method.upper()
             if method not in cls.valid_form_methods:
-                method = 'GET'
+                method = "GET"
 
         return cls(url=url, method=method, formdata=formdata, **kwargs)
 
 
 def _get_form_url(form: FormElement, url: Optional[str]) -> str:
     if url is None:
-        action = form.get('action')
+        action = form.get("action")
         if action is None:
             return form.base_url
         return urljoin(form.base_url, strip_html5_whitespace(action))
@@ -84,9 +89,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str:
 
 
 def _urlencode(seq: Iterable, enc: str) -> str:
-    values = [(to_bytes(k, enc), to_bytes(v, enc))
-              for k, vs in seq
-              for v in (vs if is_listlike(vs) else [vs])]
+    values = [
+        (to_bytes(k, enc), to_bytes(v, enc))
+        for k, vs in seq
+        for v in (vs if is_listlike(vs) else [vs])
+    ]
     return urlencode(values, doseq=True)
 
 
@@ -99,7 +106,7 @@ def _get_form(
 ) -> FormElement:
     """Find the wanted form element within the given response."""
     root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
-    forms = root.xpath('//form')
+    forms = root.xpath("//form")
     if not forms:
         raise ValueError(f"No <form> element found in {response}")
 
@@ -119,12 +126,12 @@ def _get_form(
         if nodes:
             el = nodes[0]
             while True:
-                if el.tag == 'form':
+                if el.tag == "form":
                     return el
                 el = el.getparent()
                 if el is None:
                     break
-        raise ValueError(f'No <form> element found with {formxpath}')
+        raise ValueError(f"No <form> element found with {formxpath}")
 
     # If we get here, it means that either formname was None or invalid
     if formnumber is not None:
@@ -146,19 +153,21 @@ def _get_inputs(
     try:
         formdata_keys = dict(formdata or ()).keys()
     except (ValueError, TypeError):
-        raise ValueError('formdata should be a dict or iterable of tuples')
+        raise ValueError("formdata should be a dict or iterable of tuples")
 
     if not formdata:
         formdata = []
-    inputs = form.xpath('descendant::textarea'
-                        '|descendant::select'
-                        '|descendant::input[not(@type) or @type['
-                        ' not(re:test(., "^(?:submit|image|reset)$", "i"))'
-                        ' and (../@checked or'
-                        '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
-                        namespaces={"re": "http://exslt.org/regular-expressions"})
+    inputs = form.xpath(
+        "descendant::textarea"
+        "|descendant::select"
+        "|descendant::input[not(@type) or @type["
+        ' not(re:test(., "^(?:submit|image|reset)$", "i"))'
+        " and (../@checked or"
+        '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
+        namespaces={"re": "http://exslt.org/regular-expressions"},
+    )
     values = [
-        (k, '' if v is None else v)
+        (k, "" if v is None else v)
         for k, v in (_value(e) for e in inputs)
         if k and k not in formdata_keys
     ]
@@ -178,7 +187,7 @@ def _get_inputs(
 def _value(ele: HtmlElement):
     n = ele.name
     v = ele.value
-    if ele.tag == 'select':
+    if ele.tag == "select":
         return _select_value(ele, n, v)
     return n, v
 
@@ -193,51 +202,57 @@ def _select_value(ele: SelectElement, n: str, v: str):
     if v is not None and multiple:
         # This is a workround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
-        selected_options = ele.xpath('.//option[@selected]')
-        values = [(o.get('value') or o.text or '').strip() for o in selected_options]
+        selected_options = ele.xpath(".//option[@selected]")
+        values = [(o.get("value") or o.text or "").strip() for o in selected_options]
         return n, values
     return n, v
 
 
-def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]:
+def _get_clickable(
+    clickdata: Optional[dict], form: FormElement
+) -> Optional[Tuple[str, str]]:
     """
     Returns the clickable element specified in clickdata,
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = list(form.xpath(
-        'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
-        '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
-        namespaces={"re": "http://exslt.org/regular-expressions"}
-    ))
+    clickables = list(
+        form.xpath(
+            'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
+            '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
+            namespaces={"re": "http://exslt.org/regular-expressions"},
+        )
+    )
     if not clickables:
         return None
 
     # If we don't have clickdata, we just use the first clickable element
     if clickdata is None:
         el = clickables[0]
-        return (el.get('name'), el.get('value') or '')
+        return (el.get("name"), el.get("value") or "")
 
     # If clickdata is given, we compare it to the clickable elements to find a
     # match. We first look to see if the number is specified in clickdata,
     # because that uniquely identifies the element
-    nr = clickdata.get('nr', None)
+    nr = clickdata.get("nr", None)
     if nr is not None:
         try:
             el = list(form.inputs)[nr]
         except IndexError:
             pass
         else:
-            return (el.get('name'), el.get('value') or '')
+            return (el.get("name"), el.get("value") or "")
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items())
+    xpath = ".//*" + "".join(f'[@{k}="{v}"]' for k, v in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
-        return (el[0].get('name'), el[0].get('value') or '')
+        return (el[0].get("name"), el[0].get("value") or "")
     if len(el) > 1:
-        raise ValueError(f"Multiple elements found ({el!r}) matching the "
-                         f"criteria in clickdata: {clickdata!r}")
+        raise ValueError(
+            f"Multiple elements found ({el!r}) matching the "
+            f"criteria in clickdata: {clickdata!r}"
+        )
     else:
-        raise ValueError(f'No clickable element matching clickdata: {clickdata!r}')
+        raise ValueError(f"No clickable element matching clickdata: {clickdata!r}")
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 728a2a104..35e2808c2 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -20,42 +20,44 @@ class JsonRequest(Request):
 
     def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
         dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
-        dumps_kwargs.setdefault('sort_keys', True)
+        dumps_kwargs.setdefault("sort_keys", True)
         self._dumps_kwargs = dumps_kwargs
 
-        body_passed = kwargs.get('body', None) is not None
-        data = kwargs.pop('data', None)
+        body_passed = kwargs.get("body", None) is not None
+        data = kwargs.pop("data", None)
         data_passed = data is not None
 
         if body_passed and data_passed:
-            warnings.warn('Both body and data passed. data will be ignored')
+            warnings.warn("Both body and data passed. data will be ignored")
         elif not body_passed and data_passed:
-            kwargs['body'] = self._dumps(data)
-            if 'method' not in kwargs:
-                kwargs['method'] = 'POST'
+            kwargs["body"] = self._dumps(data)
+            if "method" not in kwargs:
+                kwargs["method"] = "POST"
 
         super().__init__(*args, **kwargs)
-        self.headers.setdefault('Content-Type', 'application/json')
-        self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
+        self.headers.setdefault("Content-Type", "application/json")
+        self.headers.setdefault(
+            "Accept", "application/json, text/javascript, */*; q=0.01"
+        )
 
     @property
     def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
     def replace(self, *args, **kwargs) -> Request:
-        body_passed = kwargs.get('body', None) is not None
-        data = kwargs.pop('data', None)
+        body_passed = kwargs.get("body", None) is not None
+        data = kwargs.pop("data", None)
         data_passed = data is not None
 
         if body_passed and data_passed:
-            warnings.warn('Both body and data passed. data will be ignored')
+            warnings.warn("Both body and data passed. data will be ignored")
         elif not body_passed and data_passed:
-            kwargs['body'] = self._dumps(data)
+            kwargs["body"] = self._dumps(data)
 
         return super().replace(*args, **kwargs)
 
     def _dumps(self, data: dict) -> str:
-        """Convert to JSON """
+        """Convert to JSON"""
         return json.dumps(data, **self._dumps_kwargs)
 
 
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index 06d98cea5..c0a6e86c1 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -15,21 +15,20 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 
 class XmlRpcRequest(Request):
-
     def __init__(self, *args, encoding: Optional[str] = None, **kwargs):
-        if 'body' not in kwargs and 'params' in kwargs:
+        if "body" not in kwargs and "params" in kwargs:
             kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
-            kwargs['body'] = xmlrpclib.dumps(**kw)
+            kwargs["body"] = xmlrpclib.dumps(**kw)
 
         # spec defines that requests must use POST method
-        kwargs.setdefault('method', 'POST')
+        kwargs.setdefault("method", "POST")
 
         # xmlrpc query multiples times over the same url
-        kwargs.setdefault('dont_filter', True)
+        kwargs.setdefault("dont_filter", True)
 
         # restore encoding
         if encoding is not None:
-            kwargs['encoding'] = encoding
+            kwargs["encoding"] = encoding
 
         super().__init__(*args, **kwargs)
-        self.headers.setdefault('Content-Type', 'text/xml')
+        self.headers.setdefault("Content-Type", "text/xml")
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 7626946ec..4213d491d 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -21,7 +21,15 @@ class Response(object_ref):
     """
 
     attributes: Tuple[str, ...] = (
-        "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol",
+        "url",
+        "status",
+        "headers",
+        "body",
+        "flags",
+        "request",
+        "certificate",
+        "ip_address",
+        "protocol",
     )
     """A tuple of :class:`str` objects containing the name of all public
     attributes of the class that are also keyword parameters of the
@@ -79,26 +87,28 @@ class Response(object_ref):
         if isinstance(url, str):
             self._url = url
         else:
-            raise TypeError(f'{type(self).__name__} url must be str, '
-                            f'got {type(url).__name__}')
+            raise TypeError(
+                f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"
+            )
 
-    url = property(_get_url, obsolete_setter(_set_url, 'url'))
+    url = property(_get_url, obsolete_setter(_set_url, "url"))
 
     def _get_body(self):
         return self._body
 
     def _set_body(self, body):
         if body is None:
-            self._body = b''
+            self._body = b""
         elif not isinstance(body, bytes):
             raise TypeError(
                 "Response body must be bytes. "
                 "If you want to pass unicode body use TextResponse "
-                "or HtmlResponse.")
+                "or HtmlResponse."
+            )
         else:
             self._body = body
 
-    body = property(_get_body, obsolete_setter(_set_body, 'body'))
+    body = property(_get_body, obsolete_setter(_set_body, "body"))
 
     def __repr__(self):
         return f"<{self.status} {self.url}>"
@@ -111,7 +121,7 @@ class Response(object_ref):
         """Create a new Response with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop('cls', self.__class__)
+        cls = kwargs.pop("cls", self.__class__)
         return cls(*args, **kwargs)
 
     def urljoin(self, url):
@@ -138,9 +148,22 @@ class Response(object_ref):
         """
         raise NotSupported("Response content isn't text")
 
-    def follow(self, url, callback=None, method='GET', headers=None, body=None,
-               cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
+    def follow(
+        self,
+        url,
+        callback=None,
+        method="GET",
+        headers=None,
+        body=None,
+        cookies=None,
+        meta=None,
+        encoding="utf-8",
+        priority=0,
+        dont_filter=False,
+        errback=None,
+        cb_kwargs=None,
+        flags=None,
+    ) -> Request:
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
         It accepts the same arguments as ``Request.__init__`` method,
@@ -176,10 +199,22 @@ class Response(object_ref):
             flags=flags,
         )
 
-    def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
-                   cookies=None, meta=None, encoding='utf-8', priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None,
-                   flags=None) -> Generator[Request, None, None]:
+    def follow_all(
+        self,
+        urls,
+        callback=None,
+        method="GET",
+        headers=None,
+        body=None,
+        cookies=None,
+        meta=None,
+        encoding="utf-8",
+        priority=0,
+        dont_filter=False,
+        errback=None,
+        cb_kwargs=None,
+        flags=None,
+    ) -> Generator[Request, None, None]:
         """
         .. versionadded:: 2.0
 
@@ -192,7 +227,7 @@ class Response(object_ref):
         method which supports selectors in addition to absolute/relative URLs
         and Link objects.
         """
-        if not hasattr(urls, '__iter__'):
+        if not hasattr(urls, "__iter__"):
             raise TypeError("'urls' argument must be an iterable")
         return (
             self.follow(
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index da81d0a4a..f9df4e1b0 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -30,13 +30,13 @@ _NONE = object()
 
 class TextResponse(Response):
 
-    _DEFAULT_ENCODING = 'ascii'
+    _DEFAULT_ENCODING = "ascii"
     _cached_decoded_json = _NONE
 
     attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
 
     def __init__(self, *args, **kwargs):
-        self._encoding = kwargs.pop('encoding', None)
+        self._encoding = kwargs.pop("encoding", None)
         self._cached_benc = None
         self._cached_ubody = None
         self._cached_selector = None
@@ -49,11 +49,13 @@ class TextResponse(Response):
             super()._set_url(url)
 
     def _set_body(self, body):
-        self._body = b''  # used by encoding detection
+        self._body = b""  # used by encoding detection
         if isinstance(body, str):
             if self._encoding is None:
-                raise TypeError('Cannot convert unicode body - '
-                                f'{type(self).__name__} has no encoding')
+                raise TypeError(
+                    "Cannot convert unicode body - "
+                    f"{type(self).__name__} has no encoding"
+                )
             self._body = body.encode(self._encoding)
         else:
             super()._set_body(body)
@@ -82,12 +84,12 @@ class TextResponse(Response):
 
     @property
     def text(self):
-        """ Body as unicode """
+        """Body as unicode"""
         # access self.encoding before _cached_ubody to make sure
         # _body_inferred_encoding is called
         benc = self.encoding
         if self._cached_ubody is None:
-            charset = f'charset={benc}'
+            charset = f"charset={benc}"
             self._cached_ubody = html_to_unicode(charset, self.body)[1]
         return self._cached_ubody
 
@@ -98,21 +100,24 @@ class TextResponse(Response):
 
     @memoizemethod_noargs
     def _headers_encoding(self):
-        content_type = self.headers.get(b'Content-Type', b'')
+        content_type = self.headers.get(b"Content-Type", b"")
         return http_content_type_encoding(to_unicode(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_unicode(self.headers.get(b'Content-Type', b''))
-            benc, ubody = html_to_unicode(content_type, self.body,
-                                          auto_detect_fun=self._auto_detect_fun,
-                                          default_encoding=self._DEFAULT_ENCODING)
+            content_type = to_unicode(self.headers.get(b"Content-Type", b""))
+            benc, ubody = html_to_unicode(
+                content_type,
+                self.body,
+                auto_detect_fun=self._auto_detect_fun,
+                default_encoding=self._DEFAULT_ENCODING,
+            )
             self._cached_benc = benc
             self._cached_ubody = ubody
         return self._cached_benc
 
     def _auto_detect_fun(self, text):
-        for enc in (self._DEFAULT_ENCODING, 'utf-8', 'cp1252'):
+        for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):
             try:
                 text.decode(enc)
             except UnicodeError:
@@ -130,6 +135,7 @@ class TextResponse(Response):
     @property
     def selector(self):
         from scrapy.selector import Selector
+
         if self._cached_selector is None:
             self._cached_selector = Selector(self)
         return self._cached_selector
@@ -140,9 +146,22 @@ class TextResponse(Response):
     def css(self, query):
         return self.selector.css(query)
 
-    def follow(self, url, callback=None, method='GET', headers=None, body=None,
-               cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
+    def follow(
+        self,
+        url,
+        callback=None,
+        method="GET",
+        headers=None,
+        body=None,
+        cookies=None,
+        meta=None,
+        encoding=None,
+        priority=0,
+        dont_filter=False,
+        errback=None,
+        cb_kwargs=None,
+        flags=None,
+    ) -> Request:
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
         It accepts the same arguments as ``Request.__init__`` method,
@@ -180,10 +199,24 @@ class TextResponse(Response):
             flags=flags,
         )
 
-    def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
-                   cookies=None, meta=None, encoding=None, priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None, flags=None,
-                   css=None, xpath=None) -> Generator[Request, None, None]:
+    def follow_all(
+        self,
+        urls=None,
+        callback=None,
+        method="GET",
+        headers=None,
+        body=None,
+        cookies=None,
+        meta=None,
+        encoding=None,
+        priority=0,
+        dont_filter=False,
+        errback=None,
+        cb_kwargs=None,
+        flags=None,
+        css=None,
+        xpath=None,
+    ) -> Generator[Request, None, None]:
         """
         A generator that produces :class:`~.Request` instances to follow all
         links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
@@ -251,12 +284,13 @@ def _url_from_selector(sel):
     if isinstance(sel.root, str):
         # e.g. ::attr(href) result
         return strip_html5_whitespace(sel.root)
-    if not hasattr(sel.root, 'tag'):
+    if not hasattr(sel.root, "tag"):
         raise _InvalidSelector(f"Unsupported selector: {sel}")
-    if sel.root.tag not in ('a', 'link'):
-        raise _InvalidSelector("Only <a> and <link> elements are supported; "
-                               f"got <{sel.root.tag}>")
-    href = sel.root.get('href')
+    if sel.root.tag not in ("a", "link"):
+        raise _InvalidSelector(
+            "Only <a> and <link> elements are supported; " f"got <{sel.root.tag}>"
+        )
+    href = sel.root.get("href")
     if href is None:
         raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}")
     return strip_html5_whitespace(href)
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index 1896ec31e..9a2c5f170 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -2,7 +2,6 @@ from zope.interface import Interface
 
 
 class ISpiderLoader(Interface):
-
     def from_settings(settings):
         """Return an instance of the class for the given settings"""
 
diff --git a/scrapy/item.py b/scrapy/item.py
index 2521ac829..d3eb90b7b 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -24,11 +24,11 @@ class ItemMeta(ABCMeta):
     """
 
     def __new__(mcs, class_name, bases, attrs):
-        classcell = attrs.pop('__classcell__', None)
-        new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
-        _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
+        classcell = attrs.pop("__classcell__", None)
+        new_bases = tuple(base._class for base in bases if hasattr(base, "_class"))
+        _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
 
-        fields = getattr(_class, 'fields', {})
+        fields = getattr(_class, "fields", {})
         new_attrs = {}
         for n in dir(_class):
             v = getattr(_class, n)
@@ -37,10 +37,10 @@ class ItemMeta(ABCMeta):
             elif n in attrs:
                 new_attrs[n] = attrs[n]
 
-        new_attrs['fields'] = fields
-        new_attrs['_class'] = _class
+        new_attrs["fields"] = fields
+        new_attrs["_class"] = _class
         if classcell is not None:
-            new_attrs['__classcell__'] = classcell
+            new_attrs["__classcell__"] = classcell
         return super().__new__(mcs, class_name, bases, new_attrs)
 
 
@@ -93,7 +93,7 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta):
         raise AttributeError(name)
 
     def __setattr__(self, name, value):
-        if not name.startswith('_'):
+        if not name.startswith("_"):
             raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value")
         super().__setattr__(name, value)
 
@@ -115,6 +115,5 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta):
         return self.__class__(self)
 
     def deepcopy(self):
-        """Return a :func:`~copy.deepcopy` of this item.
-        """
+        """Return a :func:`~copy.deepcopy` of this item."""
         return deepcopy(self)
diff --git a/scrapy/link.py b/scrapy/link.py
index e70667361..704649731 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -24,9 +24,9 @@ class Link:
                     of the anchor tag.
     """
 
-    __slots__ = ['url', 'text', 'fragment', 'nofollow']
+    __slots__ = ["url", "text", "fragment", "nofollow"]
 
-    def __init__(self, url, text='', fragment='', nofollow=False):
+    def __init__(self, url, text="", fragment="", nofollow=False):
         if not isinstance(url, str):
             got = url.__class__.__name__
             raise TypeError(f"Link urls must be str objects, got {got}")
@@ -44,10 +44,12 @@ class Link:
         )
 
     def __hash__(self):
-        return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
+        return (
+            hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
+        )
 
     def __repr__(self):
         return (
-            f'Link(url={self.url!r}, text={self.text!r}, '
-            f'fragment={self.fragment!r}, nofollow={self.nofollow!r})'
+            f"Link(url={self.url!r}, text={self.text!r}, "
+            f"fragment={self.fragment!r}, nofollow={self.nofollow!r})"
         )
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index b3b1eea55..ae2948d73 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -10,25 +10,81 @@ import re
 # common file extensions that are not followed if they occur in links
 IGNORED_EXTENSIONS = [
     # archives
-    '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip',
-
+    "7z",
+    "7zip",
+    "bz2",
+    "rar",
+    "tar",
+    "tar.gz",
+    "xz",
+    "zip",
     # images
-    'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
-    'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico',
-
+    "mng",
+    "pct",
+    "bmp",
+    "gif",
+    "jpg",
+    "jpeg",
+    "png",
+    "pst",
+    "psp",
+    "tif",
+    "tiff",
+    "ai",
+    "drw",
+    "dxf",
+    "eps",
+    "ps",
+    "svg",
+    "cdr",
+    "ico",
     # audio
-    'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff',
-
+    "mp3",
+    "wma",
+    "ogg",
+    "wav",
+    "ra",
+    "aac",
+    "mid",
+    "au",
+    "aiff",
     # video
-    '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
-    'm4a', 'm4v', 'flv', 'webm',
-
+    "3gp",
+    "asf",
+    "asx",
+    "avi",
+    "mov",
+    "mp4",
+    "mpg",
+    "qt",
+    "rm",
+    "swf",
+    "wmv",
+    "m4a",
+    "m4v",
+    "flv",
+    "webm",
     # office suites
-    'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',
-    'odp',
-
+    "xls",
+    "xlsx",
+    "ppt",
+    "pptx",
+    "pps",
+    "doc",
+    "docx",
+    "odt",
+    "ods",
+    "odg",
+    "odp",
     # other
-    'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk'
+    "css",
+    "pdf",
+    "exe",
+    "bin",
+    "rss",
+    "dmg",
+    "iso",
+    "apk",
 ]
 
 
@@ -40,7 +96,7 @@ def _matches(url, regexs):
 
 
 def _is_valid_url(url):
-    return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
+    return url.split("://", 1)[0] in {"http", "https", "file", "ftp"}
 
 
 # Top-level imports
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1e6ab984a..0bd28a882 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -11,8 +11,13 @@ from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
 from scrapy.link import Link
-from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches,
-                                   _re_type, re)
+from scrapy.linkextractors import (
+    IGNORED_EXTENSIONS,
+    _is_valid_url,
+    _matches,
+    _re_type,
+    re,
+)
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
@@ -26,8 +31,8 @@ _collect_string_content = etree.XPath("string()")
 
 def _nons(tag):
     if isinstance(tag, str):
-        if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE:
-            return tag.split('}')[-1]
+        if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE:
+            return tag.split("}")[-1]
     return tag
 
 
@@ -41,14 +46,22 @@ def _canonicalize_link_url(link):
 
 class LxmlParserLinkExtractor:
     def __init__(
-        self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
+        self,
+        tag="a",
+        attr="href",
+        process=None,
+        unique=False,
+        strip=True,
+        canonicalized=False,
     ):
         self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
         self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
         self.process_attr = process if callable(process) else _identity
         self.unique = unique
         self.strip = strip
-        self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
+        self.link_key = (
+            operator.attrgetter("url") if canonicalized else _canonicalize_link_url
+        )
 
     def _iter_links(self, document):
         for el in document.iter(etree.Element):
@@ -78,17 +91,22 @@ class LxmlParserLinkExtractor:
             url = safe_url_string(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
-            link = Link(url, _collect_string_content(el) or '',
-                        nofollow=rel_has_nofollow(el.get('rel')))
+            link = Link(
+                url,
+                _collect_string_content(el) or "",
+                nofollow=rel_has_nofollow(el.get("rel")),
+            )
             links.append(link)
         return self._deduplicate_if_needed(links)
 
     def extract_links(self, response):
         base_url = get_base_url(response)
-        return self._extract_links(response.selector, response.url, response.encoding, base_url)
+        return self._extract_links(
+            response.selector, response.url, response.encoding, base_url
+        )
 
     def _process_links(self, links):
-        """ Normalize and filter extracted links
+        """Normalize and filter extracted links
 
         The subclass should override it if necessary
         """
@@ -110,8 +128,8 @@ class LxmlLinkExtractor:
         allow_domains=(),
         deny_domains=(),
         restrict_xpaths=(),
-        tags=('a', 'area'),
-        attrs=('href',),
+        tags=("a", "area"),
+        attrs=("href",),
         canonicalize=False,
         unique=True,
         process_value=None,
@@ -127,26 +145,31 @@ class LxmlLinkExtractor:
             unique=unique,
             process=process_value,
             strip=strip,
-            canonicalized=canonicalize
+            canonicalized=canonicalize,
         )
-        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
-                          for x in arg_to_iter(allow)]
-        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
-                         for x in arg_to_iter(deny)]
+        self.allow_res = [
+            x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)
+        ]
+        self.deny_res = [
+            x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)
+        ]
 
         self.allow_domains = set(arg_to_iter(allow_domains))
         self.deny_domains = set(arg_to_iter(deny_domains))
 
         self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
-        self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,
-                                          arg_to_iter(restrict_css)))
+        self.restrict_xpaths += tuple(
+            map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css))
+        )
 
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS
         self.canonicalize = canonicalize
-        self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
-        self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)
-                              for x in arg_to_iter(restrict_text)]
+        self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)}
+        self.restrict_text = [
+            x if isinstance(x, _re_type) else re.compile(x)
+            for x in arg_to_iter(restrict_text)
+        ]
 
     def _link_allowed(self, link):
         if not _is_valid_url(link.url):
@@ -156,11 +179,15 @@ class LxmlLinkExtractor:
         if self.deny_res and _matches(link.url, self.deny_res):
             return False
         parsed_url = urlparse(link.url)
-        if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):
+        if self.allow_domains and not url_is_from_any_domain(
+            parsed_url, self.allow_domains
+        ):
             return False
         if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains):
             return False
-        if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
+        if self.deny_extensions and url_has_any_extension(
+            parsed_url, self.deny_extensions
+        ):
             return False
         if self.restrict_text and not _matches(link.text, self.restrict_text):
             return False
@@ -173,7 +200,11 @@ class LxmlLinkExtractor:
         if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
             return False
 
-        allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]
+        allowed = (
+            (regex.search(url) for regex in self.allow_res)
+            if self.allow_res
+            else [True]
+        )
         denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
         return any(allowed) and not any(denied)
 
@@ -200,9 +231,7 @@ class LxmlLinkExtractor:
         base_url = get_base_url(response)
         if self.restrict_xpaths:
             docs = [
-                subdoc
-                for x in self.restrict_xpaths
-                for subdoc in response.xpath(x)
+                subdoc for x in self.restrict_xpaths for subdoc in response.xpath(x)
             ]
         else:
             docs = [response.selector]
diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py
index 3b8a6ee94..3e8644e0c 100644
--- a/scrapy/loader/common.py
+++ b/scrapy/loader/common.py
@@ -15,7 +15,7 @@ def wrap_loader_context(function, context):
         "scrapy.loader.common.wrap_loader_context has moved to a new library."
         "Please update your reference to itemloaders.common.wrap_loader_context",
         ScrapyDeprecationWarning,
-        stacklevel=2
+        stacklevel=2,
     )
 
     return common.wrap_loader_context(function, context)
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 51fbd19eb..f27a669d6 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -8,14 +8,14 @@ from itemloaders import processors
 from scrapy.utils.deprecate import create_deprecated_class
 
 
-MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)
+MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
 
-Compose = create_deprecated_class('Compose', processors.Compose)
+Compose = create_deprecated_class("Compose", processors.Compose)
 
-TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)
+TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
 
-Identity = create_deprecated_class('Identity', processors.Identity)
+Identity = create_deprecated_class("Identity", processors.Identity)
 
-SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)
+SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
 
-Join = create_deprecated_class('Join', processors.Join)
+Join = create_deprecated_class("Join", processors.Join)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 87568b2d1..e0b93d812 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -54,20 +54,20 @@ class LogFormatter:
 
     def crawled(self, request, response, spider):
         """Logs a message when the crawler finds a webpage."""
-        request_flags = f' {str(request.flags)}' if request.flags else ''
-        response_flags = f' {str(response.flags)}' if response.flags else ''
+        request_flags = f" {str(request.flags)}" if request.flags else ""
+        response_flags = f" {str(response.flags)}" if response.flags else ""
         return {
-            'level': logging.DEBUG,
-            'msg': CRAWLEDMSG,
-            'args': {
-                'status': response.status,
-                'request': request,
-                'request_flags': request_flags,
-                'referer': referer_str(request),
-                'response_flags': response_flags,
+            "level": logging.DEBUG,
+            "msg": CRAWLEDMSG,
+            "args": {
+                "status": response.status,
+                "request": request,
+                "request_flags": request_flags,
+                "referer": referer_str(request),
+                "response_flags": response_flags,
                 # backward compatibility with Scrapy logformatter below 1.4 version
-                'flags': response_flags
-            }
+                "flags": response_flags,
+            },
         }
 
     def scraped(self, item, response, spider):
@@ -77,23 +77,23 @@ class LogFormatter:
         else:
             src = response
         return {
-            'level': logging.DEBUG,
-            'msg': SCRAPEDMSG,
-            'args': {
-                'src': src,
-                'item': item,
-            }
+            "level": logging.DEBUG,
+            "msg": SCRAPEDMSG,
+            "args": {
+                "src": src,
+                "item": item,
+            },
         }
 
     def dropped(self, item, exception, response, spider):
         """Logs a message when an item is dropped while it is passing through the item pipeline."""
         return {
-            'level': logging.WARNING,
-            'msg': DROPPEDMSG,
-            'args': {
-                'exception': exception,
-                'item': item,
-            }
+            "level": logging.WARNING,
+            "msg": DROPPEDMSG,
+            "args": {
+                "exception": exception,
+                "item": item,
+            },
         }
 
     def item_error(self, item, exception, response, spider):
@@ -103,11 +103,11 @@ class LogFormatter:
         .. versionadded:: 2.0
         """
         return {
-            'level': logging.ERROR,
-            'msg': ITEMERRORMSG,
-            'args': {
-                'item': item,
-            }
+            "level": logging.ERROR,
+            "msg": ITEMERRORMSG,
+            "args": {
+                "item": item,
+            },
         }
 
     def spider_error(self, failure, request, response, spider):
@@ -116,12 +116,12 @@ class LogFormatter:
         .. versionadded:: 2.0
         """
         return {
-            'level': logging.ERROR,
-            'msg': SPIDERERRORMSG,
-            'args': {
-                'request': request,
-                'referer': referer_str(request),
-            }
+            "level": logging.ERROR,
+            "msg": SPIDERERRORMSG,
+            "args": {
+                "request": request,
+                "referer": referer_str(request),
+            },
         }
 
     def download_error(self, failure, request, spider, errmsg=None):
@@ -130,16 +130,16 @@ class LogFormatter:
 
         .. versionadded:: 2.0
         """
-        args = {'request': request}
+        args = {"request": request}
         if errmsg:
             msg = DOWNLOADERRORMSG_LONG
-            args['errmsg'] = errmsg
+            args["errmsg"] = errmsg
         else:
             msg = DOWNLOADERRORMSG_SHORT
         return {
-            'level': logging.ERROR,
-            'msg': msg,
-            'args': args,
+            "level": logging.ERROR,
+            "msg": msg,
+            "args": args,
         }
 
     @classmethod
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 2a25ccd44..fbde9c547 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -34,8 +34,15 @@ def _to_bytes_or_none(text):
 
 class MailSender:
     def __init__(
-        self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None,
-        smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False
+        self,
+        smtphost="localhost",
+        mailfrom="scrapy@localhost",
+        smtpuser=None,
+        smtppass=None,
+        smtpport=25,
+        smtptls=False,
+        smtpssl=False,
+        debug=False,
     ):
         self.smtphost = smtphost
         self.smtpport = smtpport
@@ -49,44 +56,57 @@ class MailSender:
     @classmethod
     def from_settings(cls, settings):
         return cls(
-            smtphost=settings['MAIL_HOST'],
-            mailfrom=settings['MAIL_FROM'],
-            smtpuser=settings['MAIL_USER'],
-            smtppass=settings['MAIL_PASS'],
-            smtpport=settings.getint('MAIL_PORT'),
-            smtptls=settings.getbool('MAIL_TLS'),
-            smtpssl=settings.getbool('MAIL_SSL'),
+            smtphost=settings["MAIL_HOST"],
+            mailfrom=settings["MAIL_FROM"],
+            smtpuser=settings["MAIL_USER"],
+            smtppass=settings["MAIL_PASS"],
+            smtpport=settings.getint("MAIL_PORT"),
+            smtptls=settings.getbool("MAIL_TLS"),
+            smtpssl=settings.getbool("MAIL_SSL"),
         )
 
-    def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
+    def send(
+        self,
+        to,
+        subject,
+        body,
+        cc=None,
+        attachs=(),
+        mimetype="text/plain",
+        charset=None,
+        _callback=None,
+    ):
         from twisted.internet import reactor
+
         if attachs:
             msg = MIMEMultipart()
         else:
-            msg = MIMENonMultipart(*mimetype.split('/', 1))
+            msg = MIMENonMultipart(*mimetype.split("/", 1))
 
         to = list(arg_to_iter(to))
         cc = list(arg_to_iter(cc))
 
-        msg['From'] = self.mailfrom
-        msg['To'] = COMMASPACE.join(to)
-        msg['Date'] = formatdate(localtime=True)
-        msg['Subject'] = subject
+        msg["From"] = self.mailfrom
+        msg["To"] = COMMASPACE.join(to)
+        msg["Date"] = formatdate(localtime=True)
+        msg["Subject"] = subject
         rcpts = to[:]
         if cc:
             rcpts.extend(cc)
-            msg['Cc'] = COMMASPACE.join(cc)
+            msg["Cc"] = COMMASPACE.join(cc)
 
         if charset:
             msg.set_charset(charset)
 
         if attachs:
-            msg.attach(MIMEText(body, 'plain', charset or 'us-ascii'))
+            msg.attach(MIMEText(body, "plain", charset or "us-ascii"))
             for attach_name, mimetype, f in attachs:
-                part = MIMEBase(*mimetype.split('/'))
+                part = MIMEBase(*mimetype.split("/"))
                 part.set_payload(f.read())
                 Encoders.encode_base64(part)
-                part.add_header('Content-Disposition', 'attachment', filename=attach_name)
+                part.add_header(
+                    "Content-Disposition", "attachment", filename=attach_name
+                )
                 msg.attach(part)
         else:
             msg.set_payload(body)
@@ -95,50 +115,79 @@ class MailSender:
             _callback(to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg)
 
         if self.debug:
-            logger.debug('Debug mail sent OK: To=%(mailto)s Cc=%(mailcc)s '
-                         'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
-                         {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
-                          'mailattachs': len(attachs)})
+            logger.debug(
+                "Debug mail sent OK: To=%(mailto)s Cc=%(mailcc)s "
+                'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
+                {
+                    "mailto": to,
+                    "mailcc": cc,
+                    "mailsubject": subject,
+                    "mailattachs": len(attachs),
+                },
+            )
             return
 
-        dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
+        dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
         dfd.addCallbacks(
             callback=self._sent_ok,
             errback=self._sent_failed,
             callbackArgs=[to, cc, subject, len(attachs)],
             errbackArgs=[to, cc, subject, len(attachs)],
         )
-        reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd)
+        reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
         return dfd
 
     def _sent_ok(self, result, to, cc, subject, nattachs):
-        logger.info('Mail sent OK: To=%(mailto)s Cc=%(mailcc)s '
-                    'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
-                    {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
-                     'mailattachs': nattachs})
+        logger.info(
+            "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s "
+            'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
+            {
+                "mailto": to,
+                "mailcc": cc,
+                "mailsubject": subject,
+                "mailattachs": nattachs,
+            },
+        )
 
     def _sent_failed(self, failure, to, cc, subject, nattachs):
         errstr = str(failure.value)
-        logger.error('Unable to send mail: To=%(mailto)s Cc=%(mailcc)s '
-                     'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'
-                     '- %(mailerr)s',
-                     {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
-                      'mailattachs': nattachs, 'mailerr': errstr})
+        logger.error(
+            "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s "
+            'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'
+            "- %(mailerr)s",
+            {
+                "mailto": to,
+                "mailcc": cc,
+                "mailsubject": subject,
+                "mailattachs": nattachs,
+                "mailerr": errstr,
+            },
+        )
 
     def _sendmail(self, to_addrs, msg):
         # Import twisted.mail here because it is not available in python3
         from twisted.internet import reactor
         from twisted.mail.smtp import ESMTPSenderFactory
+
         msg = BytesIO(msg)
         d = defer.Deferred()
         factory = ESMTPSenderFactory(
-            self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
-            heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
+            self.smtpuser,
+            self.smtppass,
+            self.mailfrom,
+            to_addrs,
+            msg,
+            d,
+            heloFallback=True,
+            requireAuthentication=False,
+            requireTransportSecurity=self.smtptls,
         )
         factory.noisy = False
 
         if self.smtpssl:
-            reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
+            reactor.connectSSL(
+                self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()
+            )
         else:
             reactor.connectTCP(self.smtphost, self.smtpport, factory)
 
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 431bd76dc..15f5b23e0 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -17,13 +17,15 @@ logger = logging.getLogger(__name__)
 class MiddlewareManager:
     """Base class for implementing middleware managers"""
 
-    component_name = 'foo middleware'
+    component_name = "foo middleware"
 
     def __init__(self, *middlewares: Any) -> None:
         self.middlewares = middlewares
         # Only process_spider_output and process_spider_exception can be None.
         # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
-        self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque)
+        self.methods: Dict[
+            str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]
+        ] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -44,15 +46,21 @@ class MiddlewareManager:
                 enabled.append(clspath)
             except NotConfigured as e:
                 if e.args:
-                    clsname = clspath.split('.')[-1]
-                    logger.warning("Disabled %(clsname)s: %(eargs)s",
-                                   {'clsname': clsname, 'eargs': e.args[0]},
-                                   extra={'crawler': crawler})
+                    clsname = clspath.split(".")[-1]
+                    logger.warning(
+                        "Disabled %(clsname)s: %(eargs)s",
+                        {"clsname": clsname, "eargs": e.args[0]},
+                        extra={"crawler": crawler},
+                    )
 
-        logger.info("Enabled %(componentname)ss:\n%(enabledlist)s",
-                    {'componentname': cls.component_name,
-                     'enabledlist': pprint.pformat(enabled)},
-                    extra={'crawler': crawler})
+        logger.info(
+            "Enabled %(componentname)ss:\n%(enabledlist)s",
+            {
+                "componentname": cls.component_name,
+                "enabledlist": pprint.pformat(enabled),
+            },
+            extra={"crawler": crawler},
+        )
         return cls(*middlewares)
 
     @classmethod
@@ -60,10 +68,10 @@ class MiddlewareManager:
         return cls.from_settings(crawler.settings, crawler)
 
     def _add_middleware(self, mw) -> None:
-        if hasattr(mw, 'open_spider'):
-            self.methods['open_spider'].append(mw.open_spider)
-        if hasattr(mw, 'close_spider'):
-            self.methods['close_spider'].appendleft(mw.close_spider)
+        if hasattr(mw, "open_spider"):
+            self.methods["open_spider"].append(mw.open_spider)
+        if hasattr(mw, "close_spider"):
+            self.methods["close_spider"].appendleft(mw.close_spider)
 
     def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
         methods = cast(Iterable[Callable], self.methods[methodname])
@@ -74,7 +82,7 @@ class MiddlewareManager:
         return process_chain(methods, obj, *args)
 
     def open_spider(self, spider: Spider) -> Deferred:
-        return self._process_parallel('open_spider', spider)
+        return self._process_parallel("open_spider", spider)
 
     def close_spider(self, spider: Spider) -> Deferred:
-        return self._process_parallel('close_spider', spider)
+        return self._process_parallel("close_spider", spider)
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index c42dd423e..536341fc6 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -11,16 +11,18 @@ from scrapy.utils.defer import deferred_f_from_coro_f
 
 class ItemPipelineManager(MiddlewareManager):
 
-    component_name = 'item pipeline'
+    component_name = "item pipeline"
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings.getwithbase('ITEM_PIPELINES'))
+        return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
 
     def _add_middleware(self, pipe):
         super()._add_middleware(pipe)
-        if hasattr(pipe, 'process_item'):
-            self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))
+        if hasattr(pipe, "process_item"):
+            self.methods["process_item"].append(
+                deferred_f_from_coro_f(pipe.process_item)
+            )
 
     def process_item(self, item, spider):
-        return self._process_chain('process_item', item, spider)
+        return self._process_chain("process_item", item, spider)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 51aedafe8..01a9c41fe 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -42,8 +42,8 @@ class FileException(Exception):
 
 class FSFilesStore:
     def __init__(self, basedir: str):
-        if '://' in basedir:
-            basedir = basedir.split('://', 1)[1]
+        if "://" in basedir:
+            basedir = basedir.split("://", 1)[1]
         self.basedir = basedir
         self._mkdir(Path(self.basedir))
         self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
@@ -60,13 +60,13 @@ class FSFilesStore:
         except os.error:
             return {}
 
-        with absolute_path.open('rb') as f:
+        with absolute_path.open("rb") as f:
             checksum = md5sum(f)
 
-        return {'last_modified': last_modified, 'checksum': checksum}
+        return {"last_modified": last_modified, "checksum": checksum}
 
     def _get_filesystem_path(self, path: str) -> Path:
-        path_comps = path.split('/')
+        path_comps = path.split("/")
         return Path(self.basedir, *path_comps)
 
     def _mkdir(self, dirname: Path, domain: Optional[str] = None):
@@ -86,49 +86,49 @@ class S3FilesStore:
     AWS_USE_SSL = None
     AWS_VERIFY = None
 
-    POLICY = 'private'  # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
+    POLICY = "private"  # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
     HEADERS = {
-        'Cache-Control': 'max-age=172800',
+        "Cache-Control": "max-age=172800",
     }
 
     def __init__(self, uri):
         if not is_botocore_available():
-            raise NotConfigured('missing botocore library')
+            raise NotConfigured("missing botocore library")
         import botocore.session
+
         session = botocore.session.get_session()
         self.s3_client = session.create_client(
-            's3',
+            "s3",
             aws_access_key_id=self.AWS_ACCESS_KEY_ID,
             aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
             aws_session_token=self.AWS_SESSION_TOKEN,
             endpoint_url=self.AWS_ENDPOINT_URL,
             region_name=self.AWS_REGION_NAME,
             use_ssl=self.AWS_USE_SSL,
-            verify=self.AWS_VERIFY
+            verify=self.AWS_VERIFY,
         )
         if not uri.startswith("s3://"):
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
-        self.bucket, self.prefix = uri[5:].split('/', 1)
+        self.bucket, self.prefix = uri[5:].split("/", 1)
 
     def stat_file(self, path, info):
         def _onsuccess(boto_key):
-            checksum = boto_key['ETag'].strip('"')
-            last_modified = boto_key['LastModified']
+            checksum = boto_key["ETag"].strip('"')
+            last_modified = boto_key["LastModified"]
             modified_stamp = time.mktime(last_modified.timetuple())
-            return {'checksum': checksum, 'last_modified': modified_stamp}
+            return {"checksum": checksum, "last_modified": modified_stamp}
 
         return self._get_boto_key(path).addCallback(_onsuccess)
 
     def _get_boto_key(self, path):
-        key_name = f'{self.prefix}{path}'
+        key_name = f"{self.prefix}{path}"
         return threads.deferToThread(
-            self.s3_client.head_object,
-            Bucket=self.bucket,
-            Key=key_name)
+            self.s3_client.head_object, Bucket=self.bucket, Key=key_name
+        )
 
     def persist_file(self, path, buf, info, meta=None, headers=None):
         """Upload file to S3 storage"""
-        key_name = f'{self.prefix}{path}'
+        key_name = f"{self.prefix}{path}"
         buf.seek(0)
         extra = self._headers_to_botocore_kwargs(self.HEADERS)
         if headers:
@@ -140,39 +140,41 @@ class S3FilesStore:
             Body=buf,
             Metadata={k: str(v) for k, v in (meta or {}).items()},
             ACL=self.POLICY,
-            **extra)
+            **extra,
+        )
 
     def _headers_to_botocore_kwargs(self, headers):
-        """ Convert headers to botocore keyword arguments.
-        """
+        """Convert headers to botocore keyword arguments."""
         # This is required while we need to support both boto and botocore.
-        mapping = CaselessDict({
-            'Content-Type': 'ContentType',
-            'Cache-Control': 'CacheControl',
-            'Content-Disposition': 'ContentDisposition',
-            'Content-Encoding': 'ContentEncoding',
-            'Content-Language': 'ContentLanguage',
-            'Content-Length': 'ContentLength',
-            'Content-MD5': 'ContentMD5',
-            'Expires': 'Expires',
-            'X-Amz-Grant-Full-Control': 'GrantFullControl',
-            'X-Amz-Grant-Read': 'GrantRead',
-            'X-Amz-Grant-Read-ACP': 'GrantReadACP',
-            'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
-            'X-Amz-Object-Lock-Legal-Hold': 'ObjectLockLegalHoldStatus',
-            'X-Amz-Object-Lock-Mode': 'ObjectLockMode',
-            'X-Amz-Object-Lock-Retain-Until-Date': 'ObjectLockRetainUntilDate',
-            'X-Amz-Request-Payer': 'RequestPayer',
-            'X-Amz-Server-Side-Encryption': 'ServerSideEncryption',
-            'X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id': 'SSEKMSKeyId',
-            'X-Amz-Server-Side-Encryption-Context': 'SSEKMSEncryptionContext',
-            'X-Amz-Server-Side-Encryption-Customer-Algorithm': 'SSECustomerAlgorithm',
-            'X-Amz-Server-Side-Encryption-Customer-Key': 'SSECustomerKey',
-            'X-Amz-Server-Side-Encryption-Customer-Key-Md5': 'SSECustomerKeyMD5',
-            'X-Amz-Storage-Class': 'StorageClass',
-            'X-Amz-Tagging': 'Tagging',
-            'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
-        })
+        mapping = CaselessDict(
+            {
+                "Content-Type": "ContentType",
+                "Cache-Control": "CacheControl",
+                "Content-Disposition": "ContentDisposition",
+                "Content-Encoding": "ContentEncoding",
+                "Content-Language": "ContentLanguage",
+                "Content-Length": "ContentLength",
+                "Content-MD5": "ContentMD5",
+                "Expires": "Expires",
+                "X-Amz-Grant-Full-Control": "GrantFullControl",
+                "X-Amz-Grant-Read": "GrantRead",
+                "X-Amz-Grant-Read-ACP": "GrantReadACP",
+                "X-Amz-Grant-Write-ACP": "GrantWriteACP",
+                "X-Amz-Object-Lock-Legal-Hold": "ObjectLockLegalHoldStatus",
+                "X-Amz-Object-Lock-Mode": "ObjectLockMode",
+                "X-Amz-Object-Lock-Retain-Until-Date": "ObjectLockRetainUntilDate",
+                "X-Amz-Request-Payer": "RequestPayer",
+                "X-Amz-Server-Side-Encryption": "ServerSideEncryption",
+                "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "SSEKMSKeyId",
+                "X-Amz-Server-Side-Encryption-Context": "SSEKMSEncryptionContext",
+                "X-Amz-Server-Side-Encryption-Customer-Algorithm": "SSECustomerAlgorithm",
+                "X-Amz-Server-Side-Encryption-Customer-Key": "SSECustomerKey",
+                "X-Amz-Server-Side-Encryption-Customer-Key-Md5": "SSECustomerKeyMD5",
+                "X-Amz-Storage-Class": "StorageClass",
+                "X-Amz-Tagging": "Tagging",
+                "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",
+            }
+        )
         extra = {}
         for key, value in headers.items():
             try:
@@ -188,7 +190,7 @@ class GCSFilesStore:
 
     GCS_PROJECT_ID = None
 
-    CACHE_CONTROL = 'max-age=172800'
+    CACHE_CONTROL = "max-age=172800"
 
     # The bucket's default object ACL will be applied to the object.
     # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
@@ -196,23 +198,24 @@ class GCSFilesStore:
 
     def __init__(self, uri):
         from google.cloud import storage
+
         client = storage.Client(project=self.GCS_PROJECT_ID)
-        bucket, prefix = uri[5:].split('/', 1)
+        bucket, prefix = uri[5:].split("/", 1)
         self.bucket = client.bucket(bucket)
         self.prefix = prefix
         permissions = self.bucket.test_iam_permissions(
-            ['storage.objects.get', 'storage.objects.create']
+            ["storage.objects.get", "storage.objects.create"]
         )
-        if 'storage.objects.get' not in permissions:
+        if "storage.objects.get" not in permissions:
             logger.warning(
                 "No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
                 "Checking if files are up to date will be impossible. Files will be downloaded every time.",
-                {'bucket': bucket}
+                {"bucket": bucket},
             )
-        if 'storage.objects.create' not in permissions:
+        if "storage.objects.create" not in permissions:
             logger.error(
                 "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
-                {'bucket': bucket}
+                {"bucket": bucket},
             )
 
     def stat_file(self, path, info):
@@ -220,15 +223,18 @@ class GCSFilesStore:
             if blob:
                 checksum = blob.md5_hash
                 last_modified = time.mktime(blob.updated.timetuple())
-                return {'checksum': checksum, 'last_modified': last_modified}
+                return {"checksum": checksum, "last_modified": last_modified}
             return {}
+
         blob_path = self._get_blob_path(path)
-        return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
+        return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
+            _onsuccess
+        )
 
     def _get_content_type(self, headers):
-        if headers and 'Content-Type' in headers:
-            return headers['Content-Type']
-        return 'application/octet-stream'
+        if headers and "Content-Type" in headers:
+            return headers["Content-Type"]
+        return "application/octet-stream"
 
     def _get_blob_path(self, path):
         return self.prefix + path
@@ -242,7 +248,7 @@ class GCSFilesStore:
             blob.upload_from_string,
             data=buf.getvalue(),
             content_type=self._get_content_type(headers),
-            predefined_acl=self.POLICY
+            predefined_acl=self.POLICY,
         )
 
 
@@ -261,14 +267,19 @@ class FTPFilesStore:
         self.port = int(u.port or 21)
         self.username = u.username or self.FTP_USERNAME
         self.password = u.password or self.FTP_PASSWORD
-        self.basedir = u.path.rstrip('/')
+        self.basedir = u.path.rstrip("/")
 
     def persist_file(self, path, buf, info, meta=None, headers=None):
-        path = f'{self.basedir}/{path}'
+        path = f"{self.basedir}/{path}"
         return threads.deferToThread(
-            ftp_store_file, path=path, file=buf,
-            host=self.host, port=self.port, username=self.username,
-            password=self.password, use_active_mode=self.USE_ACTIVE_MODE
+            ftp_store_file,
+            path=path,
+            file=buf,
+            host=self.host,
+            port=self.port,
+            username=self.username,
+            password=self.password,
+            use_active_mode=self.USE_ACTIVE_MODE,
         )
 
     def stat_file(self, path, info):
@@ -282,11 +293,12 @@ class FTPFilesStore:
                 file_path = f"{self.basedir}/{path}"
                 last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
                 m = hashlib.md5()
-                ftp.retrbinary(f'RETR {file_path}', m.update)
-                return {'last_modified': last_modified, 'checksum': m.hexdigest()}
+                ftp.retrbinary(f"RETR {file_path}", m.update)
+                return {"last_modified": last_modified, "checksum": m.hexdigest()}
             # The file doesn't exist
             except Exception:
                 return {}
+
         return threads.deferToThread(_stat_file, path)
 
 
@@ -312,14 +324,14 @@ class FilesPipeline(MediaPipeline):
     MEDIA_NAME = "file"
     EXPIRES = 90
     STORE_SCHEMES = {
-        '': FSFilesStore,
-        'file': FSFilesStore,
-        's3': S3FilesStore,
-        'gs': GCSFilesStore,
-        'ftp': FTPFilesStore
+        "": FSFilesStore,
+        "file": FSFilesStore,
+        "s3": S3FilesStore,
+        "gs": GCSFilesStore,
+        "ftp": FTPFilesStore,
     }
-    DEFAULT_FILES_URLS_FIELD = 'file_urls'
-    DEFAULT_FILES_RESULT_FIELD = 'files'
+    DEFAULT_FILES_URLS_FIELD = "file_urls"
+    DEFAULT_FILES_RESULT_FIELD = "files"
 
     def __init__(self, store_uri, download_func=None, settings=None):
         if not store_uri:
@@ -330,52 +342,50 @@ class FilesPipeline(MediaPipeline):
 
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
-        resolve = functools.partial(self._key_for_pipe,
-                                    base_class_name=cls_name,
-                                    settings=settings)
-        self.expires = settings.getint(
-            resolve('FILES_EXPIRES'), self.EXPIRES
+        resolve = functools.partial(
+            self._key_for_pipe, base_class_name=cls_name, settings=settings
         )
+        self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
         if not hasattr(self, "FILES_URLS_FIELD"):
             self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
         if not hasattr(self, "FILES_RESULT_FIELD"):
             self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
         self.files_urls_field = settings.get(
-            resolve('FILES_URLS_FIELD'), self.FILES_URLS_FIELD
+            resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD
         )
         self.files_result_field = settings.get(
-            resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
+            resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
         )
 
         super().__init__(download_func=download_func, settings=settings)
 
     @classmethod
     def from_settings(cls, settings):
-        s3store = cls.STORE_SCHEMES['s3']
-        s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
-        s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
-        s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
-        s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
-        s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
-        s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
-        s3store.AWS_VERIFY = settings['AWS_VERIFY']
-        s3store.POLICY = settings['FILES_STORE_S3_ACL']
+        s3store = cls.STORE_SCHEMES["s3"]
+        s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
+        s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
+        s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
+        s3store.AWS_ENDPOINT_URL = settings["AWS_ENDPOINT_URL"]
+        s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"]
+        s3store.AWS_USE_SSL = settings["AWS_USE_SSL"]
+        s3store.AWS_VERIFY = settings["AWS_VERIFY"]
+        s3store.POLICY = settings["FILES_STORE_S3_ACL"]
 
-        gcs_store = cls.STORE_SCHEMES['gs']
-        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
-        gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
+        gcs_store = cls.STORE_SCHEMES["gs"]
+        gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
+        gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None
 
-        ftp_store = cls.STORE_SCHEMES['ftp']
-        ftp_store.FTP_USERNAME = settings['FTP_USER']
-        ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
-        ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+        ftp_store = cls.STORE_SCHEMES["ftp"]
+        ftp_store.FTP_USERNAME = settings["FTP_USER"]
+        ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
+        ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")
 
-        store_uri = settings['FILES_STORE']
+        store_uri = settings["FILES_STORE"]
         return cls(store_uri, settings=settings)
 
     def _get_store(self, uri: str):
         if Path(uri).is_absolute():  # to support win32 paths like: C:\\some\dir
-            scheme = 'file'
+            scheme = "file"
         else:
             scheme = urlparse(uri).scheme
         store_cls = self.STORE_SCHEMES[scheme]
@@ -386,7 +396,7 @@ class FilesPipeline(MediaPipeline):
             if not result:
                 return  # returning None force download
 
-            last_modified = result.get('last_modified', None)
+            last_modified = result.get("last_modified", None)
             if not last_modified:
                 return  # returning None force download
 
@@ -397,25 +407,30 @@ class FilesPipeline(MediaPipeline):
 
             referer = referer_str(request)
             logger.debug(
-                'File (uptodate): Downloaded %(medianame)s from %(request)s '
-                'referred in <%(referer)s>',
-                {'medianame': self.MEDIA_NAME, 'request': request,
-                 'referer': referer},
-                extra={'spider': info.spider}
+                "File (uptodate): Downloaded %(medianame)s from %(request)s "
+                "referred in <%(referer)s>",
+                {"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
+                extra={"spider": info.spider},
             )
-            self.inc_stats(info.spider, 'uptodate')
+            self.inc_stats(info.spider, "uptodate")
 
-            checksum = result.get('checksum', None)
-            return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
+            checksum = result.get("checksum", None)
+            return {
+                "url": request.url,
+                "path": path,
+                "checksum": checksum,
+                "status": "uptodate",
+            }
 
         path = self.file_path(request, info=info, item=item)
         dfd = defer.maybeDeferred(self.store.stat_file, path, info)
         dfd.addCallbacks(_onsuccess, lambda _: None)
         dfd.addErrback(
-            lambda f:
-            logger.error(self.__class__.__name__ + '.store.stat_file',
-                         exc_info=failure_to_exc_info(f),
-                         extra={'spider': info.spider})
+            lambda f: logger.error(
+                self.__class__.__name__ + ".store.stat_file",
+                exc_info=failure_to_exc_info(f),
+                extra={"spider": info.spider},
+            )
         )
         return dfd
 
@@ -423,11 +438,15 @@ class FilesPipeline(MediaPipeline):
         if not isinstance(failure.value, IgnoreRequest):
             referer = referer_str(request)
             logger.warning(
-                'File (unknown-error): Error downloading %(medianame)s from '
-                '%(request)s referred in <%(referer)s>: %(exception)s',
-                {'medianame': self.MEDIA_NAME, 'request': request,
-                 'referer': referer, 'exception': failure.value},
-                extra={'spider': info.spider}
+                "File (unknown-error): Error downloading %(medianame)s from "
+                "%(request)s referred in <%(referer)s>: %(exception)s",
+                {
+                    "medianame": self.MEDIA_NAME,
+                    "request": request,
+                    "referer": referer,
+                    "exception": failure.value,
+                },
+                extra={"spider": info.spider},
             )
 
         raise FileException
@@ -437,29 +456,28 @@ class FilesPipeline(MediaPipeline):
 
         if response.status != 200:
             logger.warning(
-                'File (code: %(status)s): Error downloading file from '
-                '%(request)s referred in <%(referer)s>',
-                {'status': response.status,
-                 'request': request, 'referer': referer},
-                extra={'spider': info.spider}
+                "File (code: %(status)s): Error downloading file from "
+                "%(request)s referred in <%(referer)s>",
+                {"status": response.status, "request": request, "referer": referer},
+                extra={"spider": info.spider},
             )
-            raise FileException('download-error')
+            raise FileException("download-error")
 
         if not response.body:
             logger.warning(
-                'File (empty-content): Empty file from %(request)s referred '
-                'in <%(referer)s>: no-content',
-                {'request': request, 'referer': referer},
-                extra={'spider': info.spider}
+                "File (empty-content): Empty file from %(request)s referred "
+                "in <%(referer)s>: no-content",
+                {"request": request, "referer": referer},
+                extra={"spider": info.spider},
             )
-            raise FileException('empty-content')
+            raise FileException("empty-content")
 
-        status = 'cached' if 'cached' in response.flags else 'downloaded'
+        status = "cached" if "cached" in response.flags else "downloaded"
         logger.debug(
-            'File (%(status)s): Downloaded file from %(request)s referred in '
-            '<%(referer)s>',
-            {'status': status, 'request': request, 'referer': referer},
-            extra={'spider': info.spider}
+            "File (%(status)s): Downloaded file from %(request)s referred in "
+            "<%(referer)s>",
+            {"status": status, "request": request, "referer": referer},
+            extra={"spider": info.spider},
         )
         self.inc_stats(info.spider, status)
 
@@ -468,26 +486,33 @@ class FilesPipeline(MediaPipeline):
             checksum = self.file_downloaded(response, request, info, item=item)
         except FileException as exc:
             logger.warning(
-                'File (error): Error processing file from %(request)s '
-                'referred in <%(referer)s>: %(errormsg)s',
-                {'request': request, 'referer': referer, 'errormsg': str(exc)},
-                extra={'spider': info.spider}, exc_info=True
+                "File (error): Error processing file from %(request)s "
+                "referred in <%(referer)s>: %(errormsg)s",
+                {"request": request, "referer": referer, "errormsg": str(exc)},
+                extra={"spider": info.spider},
+                exc_info=True,
             )
             raise
         except Exception as exc:
             logger.error(
-                'File (unknown-error): Error processing file from %(request)s '
-                'referred in <%(referer)s>',
-                {'request': request, 'referer': referer},
-                exc_info=True, extra={'spider': info.spider}
+                "File (unknown-error): Error processing file from %(request)s "
+                "referred in <%(referer)s>",
+                {"request": request, "referer": referer},
+                exc_info=True,
+                extra={"spider": info.spider},
             )
             raise FileException(str(exc))
 
-        return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status}
+        return {
+            "url": request.url,
+            "path": path,
+            "checksum": checksum,
+            "status": status,
+        }
 
     def inc_stats(self, spider, status):
-        spider.crawler.stats.inc_value('file_count', spider=spider)
-        spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider)
+        spider.crawler.stats.inc_value("file_count", spider=spider)
+        spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider)
 
     # Overridable Interface
     def get_media_requests(self, item, info):
@@ -513,8 +538,8 @@ class FilesPipeline(MediaPipeline):
         # Handles empty and wild extensions by trying to guess the
         # mime type then extension or default to empty string otherwise
         if media_ext not in mimetypes.types_map:
-            media_ext = ''
+            media_ext = ""
             media_type = mimetypes.guess_type(request.url)[0]
             if media_type:
                 media_ext = mimetypes.guess_extension(media_type)
-        return f'full/{media_guid}{media_ext}'
+        return f"full/{media_guid}{media_ext}"
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 6a28a3b87..b8724a1cd 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -14,6 +14,7 @@ from itemadapter import ItemAdapter
 from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.pipelines.files import FileException, FilesPipeline
+
 # TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.misc import md5sum
@@ -24,7 +25,11 @@ class NoimagesDrop(DropItem):
     """Product with no images exception"""
 
     def __init__(self, *args, **kwargs):
-        warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
+        warnings.warn(
+            "The NoimagesDrop class is deprecated",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         super().__init__(*args, **kwargs)
 
 
@@ -33,11 +38,9 @@ class ImageException(FileException):
 
 
 class ImagesPipeline(FilesPipeline):
-    """Abstract pipeline that implement the image thumbnail generation logic
+    """Abstract pipeline that implement the image thumbnail generation logic"""
 
-    """
-
-    MEDIA_NAME = 'image'
+    MEDIA_NAME = "image"
 
     # Uppercase attributes kept for backward compatibility with code that subclasses
     # ImagesPipeline. They may be overridden by settings.
@@ -45,16 +48,17 @@ class ImagesPipeline(FilesPipeline):
     MIN_HEIGHT = 0
     EXPIRES = 90
     THUMBS = {}
-    DEFAULT_IMAGES_URLS_FIELD = 'image_urls'
-    DEFAULT_IMAGES_RESULT_FIELD = 'images'
+    DEFAULT_IMAGES_URLS_FIELD = "image_urls"
+    DEFAULT_IMAGES_RESULT_FIELD = "images"
 
     def __init__(self, store_uri, download_func=None, settings=None):
         try:
             from PIL import Image
+
             self._Image = Image
         except ImportError:
             raise NotConfigured(
-                'ImagesPipeline requires installing Pillow 4.0.0 or later'
+                "ImagesPipeline requires installing Pillow 4.0.0 or later"
             )
 
         super().__init__(store_uri, settings=settings, download_func=download_func)
@@ -62,12 +66,10 @@ class ImagesPipeline(FilesPipeline):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
-        resolve = functools.partial(self._key_for_pipe,
-                                    base_class_name="ImagesPipeline",
-                                    settings=settings)
-        self.expires = settings.getint(
-            resolve("IMAGES_EXPIRES"), self.EXPIRES
+        resolve = functools.partial(
+            self._key_for_pipe, base_class_name="ImagesPipeline", settings=settings
         )
+        self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
 
         if not hasattr(self, "IMAGES_RESULT_FIELD"):
             self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD
@@ -75,47 +77,39 @@ class ImagesPipeline(FilesPipeline):
             self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
 
         self.images_urls_field = settings.get(
-            resolve('IMAGES_URLS_FIELD'),
-            self.IMAGES_URLS_FIELD
+            resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD
         )
         self.images_result_field = settings.get(
-            resolve('IMAGES_RESULT_FIELD'),
-            self.IMAGES_RESULT_FIELD
-        )
-        self.min_width = settings.getint(
-            resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH
-        )
-        self.min_height = settings.getint(
-            resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT
-        )
-        self.thumbs = settings.get(
-            resolve('IMAGES_THUMBS'), self.THUMBS
+            resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD
         )
+        self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)
+        self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)
+        self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)
 
         self._deprecated_convert_image = None
 
     @classmethod
     def from_settings(cls, settings):
-        s3store = cls.STORE_SCHEMES['s3']
-        s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
-        s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
-        s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
-        s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
-        s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
-        s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
-        s3store.AWS_VERIFY = settings['AWS_VERIFY']
-        s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
+        s3store = cls.STORE_SCHEMES["s3"]
+        s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
+        s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
+        s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
+        s3store.AWS_ENDPOINT_URL = settings["AWS_ENDPOINT_URL"]
+        s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"]
+        s3store.AWS_USE_SSL = settings["AWS_USE_SSL"]
+        s3store.AWS_VERIFY = settings["AWS_VERIFY"]
+        s3store.POLICY = settings["IMAGES_STORE_S3_ACL"]
 
-        gcs_store = cls.STORE_SCHEMES['gs']
-        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
-        gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None
+        gcs_store = cls.STORE_SCHEMES["gs"]
+        gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
+        gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None
 
-        ftp_store = cls.STORE_SCHEMES['ftp']
-        ftp_store.FTP_USERNAME = settings['FTP_USER']
-        ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
-        ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+        ftp_store = cls.STORE_SCHEMES["ftp"]
+        ftp_store.FTP_USERNAME = settings["FTP_USER"]
+        ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
+        ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")
 
-        store_uri = settings['IMAGES_STORE']
+        store_uri = settings["IMAGES_STORE"]
         return cls(store_uri, settings=settings)
 
     def file_downloaded(self, response, request, info, *, item=None):
@@ -129,9 +123,12 @@ class ImagesPipeline(FilesPipeline):
                 checksum = md5sum(buf)
             width, height = image.size
             self.store.persist_file(
-                path, buf, info,
-                meta={'width': width, 'height': height},
-                headers={'Content-Type': 'image/jpeg'})
+                path,
+                buf,
+                info,
+                meta={"width": width, "height": height},
+                headers={"Content-Type": "image/jpeg"},
+            )
         return checksum
 
     def get_images(self, response, request, info, *, item=None):
@@ -140,25 +137,35 @@ class ImagesPipeline(FilesPipeline):
 
         width, height = orig_image.size
         if width < self.min_width or height < self.min_height:
-            raise ImageException("Image too small "
-                                 f"({width}x{height} < "
-                                 f"{self.min_width}x{self.min_height})")
+            raise ImageException(
+                "Image too small "
+                f"({width}x{height} < "
+                f"{self.min_width}x{self.min_height})"
+            )
 
         if self._deprecated_convert_image is None:
-            self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image)
+            self._deprecated_convert_image = "response_body" not in get_func_args(
+                self.convert_image
+            )
             if self._deprecated_convert_image:
-                warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, '
-                              'overriden method does not accept response_body argument.',
-                              category=ScrapyDeprecationWarning)
+                warnings.warn(
+                    f"{self.__class__.__name__}.convert_image() method overriden in a deprecated way, "
+                    "overriden method does not accept response_body argument.",
+                    category=ScrapyDeprecationWarning,
+                )
 
         if self._deprecated_convert_image:
             image, buf = self.convert_image(orig_image)
         else:
-            image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body))
+            image, buf = self.convert_image(
+                orig_image, response_body=BytesIO(response.body)
+            )
         yield path, image, buf
 
         for thumb_id, size in self.thumbs.items():
-            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
+            thumb_path = self.thumb_path(
+                request, thumb_id, response=response, info=info, item=item
+            )
             if self._deprecated_convert_image:
                 thumb_image, thumb_buf = self.convert_image(image, size)
             else:
@@ -167,21 +174,24 @@ class ImagesPipeline(FilesPipeline):
 
     def convert_image(self, image, size=None, response_body=None):
         if response_body is None:
-            warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, '
-                          'method called without response_body argument.',
-                          category=ScrapyDeprecationWarning, stacklevel=2)
+            warnings.warn(
+                f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
+                "method called without response_body argument.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
 
-        if image.format == 'PNG' and image.mode == 'RGBA':
-            background = self._Image.new('RGBA', image.size, (255, 255, 255))
+        if image.format == "PNG" and image.mode == "RGBA":
+            background = self._Image.new("RGBA", image.size, (255, 255, 255))
             background.paste(image, image)
-            image = background.convert('RGB')
-        elif image.mode == 'P':
+            image = background.convert("RGB")
+        elif image.mode == "P":
             image = image.convert("RGBA")
-            background = self._Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new("RGBA", image.size, (255, 255, 255))
             background.paste(image, image)
-            image = background.convert('RGB')
-        elif image.mode != 'RGB':
-            image = image.convert('RGB')
+            image = background.convert("RGB")
+        elif image.mode != "RGB":
+            image = image.convert("RGB")
 
         if size:
             image = image.copy()
@@ -193,11 +203,11 @@ class ImagesPipeline(FilesPipeline):
             except AttributeError:
                 resampling_filter = self._Image.ANTIALIAS
             image.thumbnail(size, resampling_filter)
-        elif response_body is not None and image.format == 'JPEG':
+        elif response_body is not None and image.format == "JPEG":
             return image, response_body
 
         buf = BytesIO()
-        image.save(buf, 'JPEG')
+        image.save(buf, "JPEG")
         return image, buf
 
     def get_media_requests(self, item, info):
@@ -211,8 +221,8 @@ class ImagesPipeline(FilesPipeline):
 
     def file_path(self, request, response=None, info=None, *, item=None):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
-        return f'full/{image_guid}.jpg'
+        return f"full/{image_guid}.jpg"
 
     def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
-        return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
+        return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 5308a9793..af23b4cc8 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -34,12 +34,10 @@ class MediaPipeline:
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
-        resolve = functools.partial(self._key_for_pipe,
-                                    base_class_name="MediaPipeline",
-                                    settings=settings)
-        self.allow_redirects = settings.getbool(
-            resolve('MEDIA_ALLOW_REDIRECTS'), False
+        resolve = functools.partial(
+            self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
         )
+        self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
         self._handle_statuses(self.allow_redirects)
 
         # Check if deprecated methods are being used and make them compatible
@@ -64,7 +62,8 @@ class MediaPipeline:
         if (
             not base_class_name
             or class_name == base_class_name
-            or settings and not settings.get(formatted_key)
+            or settings
+            and not settings.get(formatted_key)
         ):
             return key
         return formatted_key
@@ -113,16 +112,23 @@ class MediaPipeline:
         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
         dfd.addCallback(self._check_media_to_download, request, info, item=item)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        dfd.addErrback(lambda f: logger.error(
-            f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider})
+        dfd.addErrback(
+            lambda f: logger.error(
+                f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
+            )
         )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
     def _make_compatible(self):
         """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
         methods = [
-            "file_path", "thumb_path", "media_to_download", "media_downloaded",
-            "file_downloaded", "image_downloaded", "get_images"
+            "file_path",
+            "thumb_path",
+            "media_to_download",
+            "media_downloaded",
+            "file_downloaded",
+            "image_downloaded",
+            "get_images",
         ]
 
         for method_name in methods:
@@ -139,7 +145,7 @@ class MediaPipeline:
             if self._expects_item[func.__name__]:
                 return func(*args, **kwargs)
 
-            kwargs.pop('item', None)
+            kwargs.pop("item", None)
             return func(*args, **kwargs)
 
         return wrapper
@@ -148,19 +154,22 @@ class MediaPipeline:
         sig = signature(func)
         self._expects_item[func.__name__] = True
 
-        if 'item' not in sig.parameters:
+        if "item" not in sig.parameters:
             old_params = str(sig)[1:-1]
             new_params = old_params + ", *, item=None"
-            warn(f'{func.__name__}(self, {old_params}) is deprecated, '
-                 f'please use {func.__name__}(self, {new_params})',
-                 ScrapyDeprecationWarning, stacklevel=2)
+            warn(
+                f"{func.__name__}(self, {old_params}) is deprecated, "
+                f"please use {func.__name__}(self, {new_params})",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
             self._expects_item[func.__name__] = False
 
     def _modify_media_request(self, request):
         if self.handle_httpstatus_list:
-            request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
+            request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
         else:
-            request.meta['handle_httpstatus_all'] = True
+            request.meta["handle_httpstatus_all"] = True
 
     def _check_media_to_download(self, result, request, info, item):
         if result is not None:
@@ -169,14 +178,22 @@ class MediaPipeline:
             # this ugly code was left only to support tests. TODO: remove
             dfd = mustbe_deferred(self.download_func, request, info.spider)
             dfd.addCallbacks(
-                callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
-                errback=self.media_failed, errbackArgs=(request, info))
+                callback=self.media_downloaded,
+                callbackArgs=(request, info),
+                callbackKeywords={"item": item},
+                errback=self.media_failed,
+                errbackArgs=(request, info),
+            )
         else:
             self._modify_media_request(request)
             dfd = self.crawler.engine.download(request)
             dfd.addCallbacks(
-                callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
-                errback=self.media_failed, errbackArgs=(request, info))
+                callback=self.media_downloaded,
+                callbackArgs=(request, info),
+                callbackKeywords={"item": item},
+                errback=self.media_failed,
+                errbackArgs=(request, info),
+            )
         return dfd
 
     def _cache_result_and_execute_waiters(self, result, fp, info):
@@ -207,9 +224,9 @@ class MediaPipeline:
             #
             # This problem does not occur in Python 2.7 since we don't have
             # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
-            context = getattr(result.value, '__context__', None)
+            context = getattr(result.value, "__context__", None)
             if isinstance(context, StopIteration):
-                setattr(result.value, '__context__', None)
+                setattr(result.value, "__context__", None)
 
         info.downloading.remove(fp)
         info.downloaded[fp] = result  # cache result
@@ -239,10 +256,10 @@ class MediaPipeline:
             for ok, value in results:
                 if not ok:
                     logger.error(
-                        '%(class)s found errors processing %(item)s',
-                        {'class': self.__class__.__name__, 'item': item},
+                        "%(class)s found errors processing %(item)s",
+                        {"class": self.__class__.__name__, "item": item},
                         exc_info=failure_to_exc_info(value),
-                        extra={'spider': info.spider}
+                        extra={"spider": info.spider},
                     )
         return item
 
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index b4b63e7c7..6f65184e5 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -18,11 +18,11 @@ def _path_safe(text):
     >>> _path_safe('some@symbol?').startswith('some_symbol_')
     True
     """
-    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text])
+    pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])
     # as we replace some letters we can get collision for different slots
     # add we add unique part
-    unique_slot = hashlib.md5(text.encode('utf8')).hexdigest()
-    return '-'.join([pathable_slot, unique_slot])
+    unique_slot = hashlib.md5(text.encode("utf8")).hexdigest()
+    return "-".join([pathable_slot, unique_slot])
 
 
 class ScrapyPriorityQueue:
@@ -77,7 +77,7 @@ class ScrapyPriorityQueue:
             self.downstream_queue_cls,
             None,
             self.crawler,
-            self.key + '/' + str(key),
+            self.key + "/" + str(key),
         )
 
     def priority(self, request):
@@ -128,7 +128,6 @@ class ScrapyPriorityQueue:
 
 
 class DownloaderInterface:
-
     def __init__(self, crawler):
         self.downloader = crawler.engine.downloader
 
@@ -139,14 +138,14 @@ class DownloaderInterface:
         return self.downloader._get_slot_key(request, None)
 
     def _active_downloads(self, slot):
-        """ Return a number of requests in a Downloader for a given slot """
+        """Return a number of requests in a Downloader for a given slot"""
         if slot not in self.downloader.slots:
             return 0
         return len(self.downloader.slots[slot].active)
 
 
 class DownloaderAwarePriorityQueue:
-    """ PriorityQueue which takes Downloader activity into account:
+    """PriorityQueue which takes Downloader activity into account:
     domains (slots) with the least amount of active downloads are dequeued
     first.
     """
@@ -156,17 +155,21 @@ class DownloaderAwarePriorityQueue:
         return cls(crawler, downstream_queue_cls, key, startprios)
 
     def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
-        if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
-            raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP')
+        if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
+            raise ValueError(
+                f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
+            )
 
         if slot_startprios and not isinstance(slot_startprios, dict):
-            raise ValueError("DownloaderAwarePriorityQueue accepts "
-                             "``slot_startprios`` as a dict; "
-                             f"{slot_startprios.__class__!r} instance "
-                             "is passed. Most likely, it means the state is"
-                             "created by an incompatible priority queue. "
-                             "Only a crawl started with the same priority "
-                             "queue class can be resumed.")
+            raise ValueError(
+                "DownloaderAwarePriorityQueue accepts "
+                "``slot_startprios`` as a dict; "
+                f"{slot_startprios.__class__!r} instance "
+                "is passed. Most likely, it means the state is"
+                "created by an incompatible priority queue. "
+                "Only a crawl started with the same priority "
+                "queue class can be resumed."
+            )
 
         self._downloader_interface = DownloaderInterface(crawler)
         self.downstream_queue_cls = downstream_queue_cls
@@ -181,7 +184,7 @@ class DownloaderAwarePriorityQueue:
         return ScrapyPriorityQueue(
             self.crawler,
             self.downstream_queue_cls,
-            self.key + '/' + _path_safe(slot),
+            self.key + "/" + _path_safe(slot),
             startprios,
         )
 
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 0bef555a6..f5d2b8e05 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,6 +1,11 @@
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple
+from twisted.internet.interfaces import (
+    IHostResolution,
+    IHostnameResolver,
+    IResolutionReceiver,
+    IResolverSimple,
+)
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
@@ -23,11 +28,11 @@ class CachingThreadedResolver(ThreadedResolver):
 
     @classmethod
     def from_crawler(cls, crawler, reactor):
-        if crawler.settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+        if crawler.settings.getbool("DNSCACHE_ENABLED"):
+            cache_size = crawler.settings.getint("DNSCACHE_SIZE")
         else:
             cache_size = 0
-        return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
+        return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT"))
 
     def install_on_reactor(self):
         self.reactor.installResolver(self)
@@ -94,8 +99,8 @@ class CachingHostnameResolver:
 
     @classmethod
     def from_crawler(cls, crawler, reactor):
-        if crawler.settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+        if crawler.settings.getbool("DNSCACHE_ENABLED"):
+            cache_size = crawler.settings.getint("DNSCACHE_SIZE")
         else:
             cache_size = 0
         return cls(reactor, cache_size)
@@ -104,7 +109,12 @@ class CachingHostnameResolver:
         self.reactor.installNameResolver(self)
 
     def resolveHostName(
-        self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP"
+        self,
+        resolutionReceiver,
+        hostName,
+        portNumber=0,
+        addressTypes=None,
+        transportSemantics="TCP",
     ):
         try:
             addresses = dnscache[hostName]
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index a34d7d25f..6b489bd8b 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -14,26 +14,26 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 class ResponseTypes:
 
     CLASSES = {
-        'text/html': 'scrapy.http.HtmlResponse',
-        'application/atom+xml': 'scrapy.http.XmlResponse',
-        'application/rdf+xml': 'scrapy.http.XmlResponse',
-        'application/rss+xml': 'scrapy.http.XmlResponse',
-        'application/xhtml+xml': 'scrapy.http.HtmlResponse',
-        'application/vnd.wap.xhtml+xml': 'scrapy.http.HtmlResponse',
-        'application/xml': 'scrapy.http.XmlResponse',
-        'application/json': 'scrapy.http.TextResponse',
-        'application/x-json': 'scrapy.http.TextResponse',
-        'application/json-amazonui-streaming': 'scrapy.http.TextResponse',
-        'application/javascript': 'scrapy.http.TextResponse',
-        'application/x-javascript': 'scrapy.http.TextResponse',
-        'text/xml': 'scrapy.http.XmlResponse',
-        'text/*': 'scrapy.http.TextResponse',
+        "text/html": "scrapy.http.HtmlResponse",
+        "application/atom+xml": "scrapy.http.XmlResponse",
+        "application/rdf+xml": "scrapy.http.XmlResponse",
+        "application/rss+xml": "scrapy.http.XmlResponse",
+        "application/xhtml+xml": "scrapy.http.HtmlResponse",
+        "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse",
+        "application/xml": "scrapy.http.XmlResponse",
+        "application/json": "scrapy.http.TextResponse",
+        "application/x-json": "scrapy.http.TextResponse",
+        "application/json-amazonui-streaming": "scrapy.http.TextResponse",
+        "application/javascript": "scrapy.http.TextResponse",
+        "application/x-javascript": "scrapy.http.TextResponse",
+        "text/xml": "scrapy.http.XmlResponse",
+        "text/*": "scrapy.http.TextResponse",
     }
 
     def __init__(self):
         self.classes = {}
         self.mimetypes = MimeTypes()
-        mimedata = get_data('scrapy', 'mime.types').decode('utf8')
+        mimedata = get_data("scrapy", "mime.types").decode("utf8")
         self.mimetypes.readfp(StringIO(mimedata))
         for mimetype, cls in self.CLASSES.items():
             self.classes[mimetype] = load_object(cls)
@@ -49,17 +49,20 @@ class ResponseTypes:
 
     def from_content_type(self, content_type, content_encoding=None):
         """Return the most appropriate Response class from an HTTP Content-Type
-        header """
+        header"""
         if content_encoding:
             return Response
-        mimetype = to_unicode(content_type).split(';')[0].strip().lower()
+        mimetype = to_unicode(content_type).split(";")[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_unicode(
-                content_disposition, encoding='latin-1', errors='replace'
-            ).split(';')[1].split('=')[1].strip('"\'')
+            filename = (
+                to_unicode(content_disposition, encoding="latin-1", errors="replace")
+                .split(";")[1]
+                .split("=")[1]
+                .strip("\"'")
+            )
             return self.from_filename(filename)
         except IndexError:
             return Response
@@ -68,13 +71,13 @@ class ResponseTypes:
         """Return the most appropriate Response class by looking at the HTTP
         headers"""
         cls = Response
-        if b'Content-Type' in headers:
+        if b"Content-Type" in headers:
             cls = self.from_content_type(
-                content_type=headers[b'Content-Type'],
-                content_encoding=headers.get(b'Content-Encoding')
+                content_type=headers[b"Content-Type"],
+                content_encoding=headers.get(b"Content-Encoding"),
             )
-        if cls is Response and b'Content-Disposition' in headers:
-            cls = self.from_content_disposition(headers[b'Content-Disposition'])
+        if cls is Response and b"Content-Disposition" in headers:
+            cls = self.from_content_disposition(headers[b"Content-Disposition"])
         return cls
 
     def from_filename(self, filename):
@@ -92,15 +95,15 @@ class ResponseTypes:
         chunk = body[:5000]
         chunk = to_bytes(chunk)
         if not binary_is_text(chunk):
-            return self.from_mimetype('application/octet-stream')
+            return self.from_mimetype("application/octet-stream")
         lowercase_chunk = chunk.lower()
         if b"<html>" in lowercase_chunk:
-            return self.from_mimetype('text/html')
+            return self.from_mimetype("text/html")
         if b"<?xml" in lowercase_chunk:
-            return self.from_mimetype('text/xml')
-        if b'<!doctype html>' in lowercase_chunk:
-            return self.from_mimetype('text/html')
-        return self.from_mimetype('text')
+            return self.from_mimetype("text/xml")
+        if b"<!doctype html>" in lowercase_chunk:
+            return self.from_mimetype("text/html")
+        return self.from_mimetype("text")
 
     def from_args(self, headers=None, url=None, filename=None, body=None):
         """Guess the most appropriate Response class based on
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 1d40f0484..0dadeef92 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -12,7 +12,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         if to_native_str_type:
             robotstxt_body = to_unicode(robotstxt_body)
         else:
-            robotstxt_body = robotstxt_body.decode('utf-8')
+            robotstxt_body = robotstxt_body.decode("utf-8")
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.
@@ -20,9 +20,9 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
             "Failure while parsing robots.txt. File either contains garbage or "
             "is in an encoding other than UTF-8, treating it as an empty file.",
             exc_info=sys.exc_info(),
-            extra={'spider': spider},
+            extra={"spider": spider},
         )
-        robotstxt_body = ''
+        robotstxt_body = ""
     return robotstxt_body
 
 
@@ -57,8 +57,11 @@ class RobotParser(metaclass=ABCMeta):
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
         from urllib.robotparser import RobotFileParser
+
         self.spider = spider
-        robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
+        robotstxt_body = decode_robotstxt(
+            robotstxt_body, spider, to_native_str_type=True
+        )
         self.rp = RobotFileParser()
         self.rp.parse(robotstxt_body.splitlines())
 
@@ -77,8 +80,9 @@ class PythonRobotParser(RobotParser):
 class ReppyRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
         from reppy.robots import Robots
+
         self.spider = spider
-        self.rp = Robots.parse('', robotstxt_body)
+        self.rp = Robots.parse("", robotstxt_body)
 
     @classmethod
     def from_crawler(cls, crawler, robotstxt_body):
@@ -93,6 +97,7 @@ class ReppyRobotParser(RobotParser):
 class RerpRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
         from robotexclusionrulesparser import RobotExclusionRulesParser
+
         self.spider = spider
         self.rp = RobotExclusionRulesParser()
         robotstxt_body = decode_robotstxt(robotstxt_body, spider)
@@ -113,6 +118,7 @@ class RerpRobotParser(RobotParser):
 class ProtegoRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
         from protego import Protego
+
         self.spider = spider
         robotstxt_body = decode_robotstxt(robotstxt_body, spider)
         self.rp = Protego.parse(robotstxt_body)
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 08f08e8d7..6ba87428e 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -8,19 +8,18 @@ from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
 
 
-__all__ = ['Selector', 'SelectorList']
+__all__ = ["Selector", "SelectorList"]
 
 
 def _st(response, st):
     if st is None:
-        return 'xml' if isinstance(response, XmlResponse) else 'html'
+        return "xml" if isinstance(response, XmlResponse) else "html"
     return st
 
 
 def _response_from_text(text, st):
-    rt = XmlResponse if st == 'xml' else HtmlResponse
-    return rt(url='about:blank', encoding='utf-8',
-              body=to_bytes(text, 'utf-8'))
+    rt = XmlResponse if st == "xml" else HtmlResponse
+    return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
 
 
 class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
@@ -61,13 +60,15 @@ class Selector(_ParselSelector, object_ref):
     detection will occur.
     """
 
-    __slots__ = ['response']
+    __slots__ = ["response"]
     selectorlist_cls = SelectorList
 
     def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
         if response is not None and text is not None:
-            raise ValueError(f'{self.__class__.__name__}.__init__() received '
-                             'both response and text')
+            raise ValueError(
+                f"{self.__class__.__name__}.__init__() received "
+                "both response and text"
+            )
 
         st = _st(response, type)
 
@@ -76,7 +77,7 @@ class Selector(_ParselSelector, object_ref):
 
         if response is not None:
             text = response.text
-            kwargs.setdefault('base_url', response.url)
+            kwargs.setdefault("base_url", response.url)
 
         self.response = response
         super().__init__(text=text, type=st, root=root, **kwargs)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b540e6182..c0d0741c5 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -8,11 +8,11 @@ from scrapy.settings import default_settings
 
 
 SETTINGS_PRIORITIES = {
-    'default': 0,
-    'command': 10,
-    'project': 20,
-    'spider': 30,
-    'cmdline': 40,
+    "default": 0,
+    "command": 10,
+    "project": 20,
+    "spider": 30,
+    "cmdline": 40,
 }
 
 
@@ -76,7 +76,7 @@ class BaseSettings(MutableMapping):
     highest priority will be retrieved.
     """
 
-    def __init__(self, values=None, priority='project'):
+    def __init__(self, values=None, priority="project"):
         self.frozen = False
         self.attributes = {}
         if values:
@@ -126,9 +126,11 @@ class BaseSettings(MutableMapping):
                 return True
             if got in ("False", "false"):
                 return False
-            raise ValueError("Supported values for boolean settings "
-                             "are 0/1, True/False, '0'/'1', "
-                             "'True'/'False' and 'true'/'false'")
+            raise ValueError(
+                "Supported values for boolean settings "
+                "are 0/1, True/False, '0'/'1', "
+                "'True'/'False' and 'true'/'false'"
+            )
 
     def getint(self, name, default=0):
         """
@@ -170,7 +172,7 @@ class BaseSettings(MutableMapping):
         """
         value = self.get(name, default or [])
         if isinstance(value, str):
-            value = value.split(',')
+            value = value.split(",")
         return list(value)
 
     def getdict(self, name, default=None):
@@ -223,7 +225,7 @@ class BaseSettings(MutableMapping):
             try:
                 return json.loads(value)
             except ValueError:
-                return value.split(',')
+                return value.split(",")
         return copy.deepcopy(value)
 
     def getwithbase(self, name):
@@ -234,7 +236,7 @@ class BaseSettings(MutableMapping):
         :type name: str
         """
         compbs = BaseSettings()
-        compbs.update(self[name + '_BASE'])
+        compbs.update(self[name + "_BASE"])
         compbs.update(self[name])
         return compbs
 
@@ -259,12 +261,12 @@ class BaseSettings(MutableMapping):
         """
         if len(self) > 0:
             return max(self.getpriority(name) for name in self)
-        return get_settings_priority('default')
+        return get_settings_priority("default")
 
     def __setitem__(self, name, value):
         self.set(name, value)
 
-    def set(self, name, value, priority='project'):
+    def set(self, name, value, priority="project"):
         """
         Store a key/value attribute with a given priority.
 
@@ -292,10 +294,10 @@ class BaseSettings(MutableMapping):
         else:
             self.attributes[name].set(value, priority)
 
-    def setdict(self, values, priority='project'):
+    def setdict(self, values, priority="project"):
         self.update(values, priority)
 
-    def setmodule(self, module, priority='project'):
+    def setmodule(self, module, priority="project"):
         """
         Store settings from a module with a given priority.
 
@@ -317,7 +319,7 @@ class BaseSettings(MutableMapping):
             if key.isupper():
                 self.set(key, getattr(module, key), priority)
 
-    def update(self, values, priority='project'):
+    def update(self, values, priority="project"):
         """
         Store key/value pairs with a given priority.
 
@@ -350,7 +352,7 @@ class BaseSettings(MutableMapping):
                 for name, value in values.items():
                     self.set(name, value, priority)
 
-    def delete(self, name, priority='project'):
+    def delete(self, name, priority="project"):
         self._assert_mutability()
         priority = get_settings_priority(priority)
         if priority >= self.getpriority(name):
@@ -403,12 +405,17 @@ class BaseSettings(MutableMapping):
         return len(self.attributes)
 
     def _to_dict(self):
-        return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
-                for k, v in self.items()}
+        return {
+            self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
+            for k, v in self.items()
+        }
 
     def _get_key(self, key_value):
-        return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
-                else str(key_value))
+        return (
+            key_value
+            if isinstance(key_value, (bool, float, int, str, type(None)))
+            else str(key_value)
+        )
 
     def copy_to_dict(self):
         """
@@ -444,17 +451,17 @@ class Settings(BaseSettings):
     described on :ref:`topics-settings-ref` already populated.
     """
 
-    def __init__(self, values=None, priority='project'):
+    def __init__(self, values=None, priority="project"):
         # Do not pass kwarg values here. We don't want to promote user-defined
         # dicts, and we want to update, not replace, default dicts with the
         # values given by the user
         super().__init__()
-        self.setmodule(default_settings, 'default')
+        self.setmodule(default_settings, "default")
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
         for name, val in self.items():
             if isinstance(val, dict):
-                self.set(name, BaseSettings(val, 'default'), 'default')
+                self.set(name, BaseSettings(val, "default"), "default")
         self.update(values, priority)
 
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5e7379b89..260ec1701 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -27,14 +27,14 @@ AUTOTHROTTLE_MAX_DELAY = 60.0
 AUTOTHROTTLE_START_DELAY = 5.0
 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 
-BOT_NAME = 'scrapybot'
+BOT_NAME = "scrapybot"
 
 CLOSESPIDER_TIMEOUT = 0
 CLOSESPIDER_PAGECOUNT = 0
 CLOSESPIDER_ITEMCOUNT = 0
 CLOSESPIDER_ERRORCOUNT = 0
 
-COMMANDS_MODULE = ''
+COMMANDS_MODULE = ""
 
 COMPRESSION_ENABLED = True
 
@@ -47,11 +47,11 @@ CONCURRENT_REQUESTS_PER_IP = 0
 COOKIES_ENABLED = True
 COOKIES_DEBUG = False
 
-DEFAULT_ITEM_CLASS = 'scrapy.item.Item'
+DEFAULT_ITEM_CLASS = "scrapy.item.Item"
 
 DEFAULT_REQUEST_HEADERS = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Language': 'en',
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en",
 }
 
 DEPTH_LIMIT = 0
@@ -60,78 +60,82 @@ DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True
 DNSCACHE_SIZE = 10000
-DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
+DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"
 DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0
 
 DOWNLOAD_HANDLERS = {}
 DOWNLOAD_HANDLERS_BASE = {
-    'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
-    'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
-    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
-    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
-    's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
-    'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
+    "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
+    "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",
+    "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
+    "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
+    "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
+    "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
 }
 
-DOWNLOAD_TIMEOUT = 180      # 3mins
+DOWNLOAD_TIMEOUT = 180  # 3mins
 
-DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024   # 1024m
-DOWNLOAD_WARNSIZE = 32 * 1024 * 1024    # 32m
+DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024  # 1024m
+DOWNLOAD_WARNSIZE = 32 * 1024 * 1024  # 32m
 
 DOWNLOAD_FAIL_ON_DATALOSS = True
 
-DOWNLOADER = 'scrapy.core.downloader.Downloader'
+DOWNLOADER = "scrapy.core.downloader.Downloader"
 
-DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
-DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
-DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
+DOWNLOADER_HTTPCLIENTFACTORY = (
+    "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
+)
+DOWNLOADER_CLIENTCONTEXTFACTORY = (
+    "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory"
+)
+DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
 # Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
-DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
+DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
 DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
 
 DOWNLOADER_MIDDLEWARES = {}
 
 DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
-    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
-    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
-    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
-    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
-    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
-    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
-    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
-    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
-    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
-    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
-    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
-    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
-    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
-    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
+    "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
+    "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
+    "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
+    "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
+    "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
+    "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
+    "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560,
+    "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
+    "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
+    "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
+    "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700,
+    "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
+    "scrapy.downloadermiddlewares.stats.DownloaderStats": 850,
+    "scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900,
     # Downloader side
 }
 
 DOWNLOADER_STATS = True
 
-DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'
+DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"
 
-EDITOR = 'vi'
-if sys.platform == 'win32':
-    EDITOR = '%s -m idlelib.idle'
+EDITOR = "vi"
+if sys.platform == "win32":
+    EDITOR = "%s -m idlelib.idle"
 
 EXTENSIONS = {}
 
 EXTENSIONS_BASE = {
-    'scrapy.extensions.corestats.CoreStats': 0,
-    'scrapy.extensions.telnet.TelnetConsole': 0,
-    'scrapy.extensions.memusage.MemoryUsage': 0,
-    'scrapy.extensions.memdebug.MemoryDebugger': 0,
-    'scrapy.extensions.closespider.CloseSpider': 0,
-    'scrapy.extensions.feedexport.FeedExporter': 0,
-    'scrapy.extensions.logstats.LogStats': 0,
-    'scrapy.extensions.spiderstate.SpiderState': 0,
-    'scrapy.extensions.throttle.AutoThrottle': 0,
+    "scrapy.extensions.corestats.CoreStats": 0,
+    "scrapy.extensions.telnet.TelnetConsole": 0,
+    "scrapy.extensions.memusage.MemoryUsage": 0,
+    "scrapy.extensions.memdebug.MemoryDebugger": 0,
+    "scrapy.extensions.closespider.CloseSpider": 0,
+    "scrapy.extensions.feedexport.FeedExporter": 0,
+    "scrapy.extensions.logstats.LogStats": 0,
+    "scrapy.extensions.spiderstate.SpiderState": 0,
+    "scrapy.extensions.throttle.AutoThrottle": 0,
 }
 
 FEED_TEMPDIR = None
@@ -142,71 +146,71 @@ FEED_EXPORT_ENCODING = None
 FEED_EXPORT_FIELDS = None
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
-    '': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'file': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
-    'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
-    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
-    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
+    "": "scrapy.extensions.feedexport.FileFeedStorage",
+    "file": "scrapy.extensions.feedexport.FileFeedStorage",
+    "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
+    "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
+    "s3": "scrapy.extensions.feedexport.S3FeedStorage",
+    "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
 }
 FEED_EXPORT_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
-    'json': 'scrapy.exporters.JsonItemExporter',
-    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
-    'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
-    'jl': 'scrapy.exporters.JsonLinesItemExporter',
-    'csv': 'scrapy.exporters.CsvItemExporter',
-    'xml': 'scrapy.exporters.XmlItemExporter',
-    'marshal': 'scrapy.exporters.MarshalItemExporter',
-    'pickle': 'scrapy.exporters.PickleItemExporter',
+    "json": "scrapy.exporters.JsonItemExporter",
+    "jsonlines": "scrapy.exporters.JsonLinesItemExporter",
+    "jsonl": "scrapy.exporters.JsonLinesItemExporter",
+    "jl": "scrapy.exporters.JsonLinesItemExporter",
+    "csv": "scrapy.exporters.CsvItemExporter",
+    "xml": "scrapy.exporters.XmlItemExporter",
+    "marshal": "scrapy.exporters.MarshalItemExporter",
+    "pickle": "scrapy.exporters.PickleItemExporter",
 }
 FEED_EXPORT_INDENT = 0
 
 FEED_STORAGE_FTP_ACTIVE = False
-FEED_STORAGE_GCS_ACL = ''
-FEED_STORAGE_S3_ACL = ''
+FEED_STORAGE_GCS_ACL = ""
+FEED_STORAGE_S3_ACL = ""
 
-FILES_STORE_S3_ACL = 'private'
-FILES_STORE_GCS_ACL = ''
+FILES_STORE_S3_ACL = "private"
+FILES_STORE_GCS_ACL = ""
 
-FTP_USER = 'anonymous'
-FTP_PASSWORD = 'guest'
+FTP_USER = "anonymous"
+FTP_PASSWORD = "guest"
 FTP_PASSIVE_MODE = True
 
 GCS_PROJECT_ID = None
 
 HTTPCACHE_ENABLED = False
-HTTPCACHE_DIR = 'httpcache'
+HTTPCACHE_DIR = "httpcache"
 HTTPCACHE_IGNORE_MISSING = False
-HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
-HTTPCACHE_IGNORE_SCHEMES = ['file']
+HTTPCACHE_IGNORE_SCHEMES = ["file"]
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'dbm'
-HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
+HTTPCACHE_DBM_MODULE = "dbm"
+HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
 HTTPCACHE_GZIP = False
 
 HTTPPROXY_ENABLED = True
-HTTPPROXY_AUTH_ENCODING = 'latin-1'
+HTTPPROXY_AUTH_ENCODING = "latin-1"
 
-IMAGES_STORE_S3_ACL = 'private'
-IMAGES_STORE_GCS_ACL = ''
+IMAGES_STORE_S3_ACL = "private"
+IMAGES_STORE_GCS_ACL = ""
 
-ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
+ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
 
 ITEM_PIPELINES = {}
 ITEM_PIPELINES_BASE = {}
 
 LOG_ENABLED = True
-LOG_ENCODING = 'utf-8'
-LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
-LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
-LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
+LOG_ENCODING = "utf-8"
+LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
+LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S"
 LOG_STDOUT = False
-LOG_LEVEL = 'DEBUG'
+LOG_LEVEL = "DEBUG"
 LOG_FILE = None
 LOG_FILE_APPEND = True
 LOG_SHORT_NAMES = False
@@ -215,14 +219,14 @@ SCHEDULER_DEBUG = False
 
 LOGSTATS_INTERVAL = 60.0
 
-MAIL_HOST = 'localhost'
+MAIL_HOST = "localhost"
 MAIL_PORT = 25
-MAIL_FROM = 'scrapy@localhost'
+MAIL_FROM = "scrapy@localhost"
 MAIL_PASS = None
 MAIL_USER = None
 
-MEMDEBUG_ENABLED = False        # enable memory debugging
-MEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown
+MEMDEBUG_ENABLED = False  # enable memory debugging
+MEMDEBUG_NOTIFY = []  # send memory debugging report by mail at engine shutdown
 
 MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
 MEMUSAGE_ENABLED = True
@@ -234,7 +238,7 @@ METAREFRESH_ENABLED = True
 METAREFRESH_IGNORE_TAGS = []
 METAREFRESH_MAXDELAY = 100
 
-NEWSPIDER_MODULE = ''
+NEWSPIDER_MODULE = ""
 
 RANDOMIZE_DOWNLOAD_DELAY = True
 
@@ -245,10 +249,10 @@ REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
 REFERER_ENABLED = True
-REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
+REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
 
-REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
-REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6'
+REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6"
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
@@ -256,39 +260,39 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
-ROBOTSTXT_PARSER = 'scrapy.robotstxt.ProtegoRobotParser'
+ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"
 ROBOTSTXT_USER_AGENT = None
 
-SCHEDULER = 'scrapy.core.scheduler.Scheduler'
-SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
-SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
-SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'
+SCHEDULER = "scrapy.core.scheduler.Scheduler"
+SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"
+SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"
+SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"
 
 SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
 
-SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
+SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"
 SPIDER_LOADER_WARN_ONLY = False
 
 SPIDER_MIDDLEWARES = {}
 
 SPIDER_MIDDLEWARES_BASE = {
     # Engine side
-    'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
-    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
-    'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
-    'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
-    'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
+    "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
+    "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
+    "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
+    "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
+    "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
     # Spider side
 }
 
 SPIDER_MODULES = []
 
-STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'
+STATS_CLASS = "scrapy.statscollectors.MemoryStatsCollector"
 STATS_DUMP = True
 
 STATSMAILER_RCPTS = []
 
-TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve())
+TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
 
 URLLENGTH_LIMIT = 2083
 
@@ -296,16 +300,16 @@ USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org
 
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
-TELNETCONSOLE_HOST = '127.0.0.1'
-TELNETCONSOLE_USERNAME = 'scrapy'
+TELNETCONSOLE_HOST = "127.0.0.1"
+TELNETCONSOLE_USERNAME = "scrapy"
 TELNETCONSOLE_PASSWORD = None
 
 TWISTED_REACTOR = None
 
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
-    'scrapy.contracts.default.UrlContract': 1,
-    'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1,
-    'scrapy.contracts.default.ReturnsContract': 2,
-    'scrapy.contracts.default.ScrapesContract': 3,
+    "scrapy.contracts.default.UrlContract": 1,
+    "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
+    "scrapy.contracts.default.ReturnsContract": 2,
+    "scrapy.contracts.default.ScrapesContract": 3,
 }
diff --git a/scrapy/shell.py b/scrapy/shell.py
index f2dff2ae3..18be383c1 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -30,7 +30,7 @@ class Shell:
     def __init__(self, crawler, update_vars=None, code=None):
         self.crawler = crawler
         self.update_vars = update_vars or (lambda x: None)
-        self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS'])
+        self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"])
         self.spider = None
         self.inthread = not threadable.isInIOThread()
         self.code = code
@@ -61,19 +61,20 @@ class Shell:
             shell = python
             """
             cfg = get_config()
-            section, option = 'settings', 'shell'
-            env = os.environ.get('SCRAPY_PYTHON_SHELL')
+            section, option = "settings", "shell"
+            env = os.environ.get("SCRAPY_PYTHON_SHELL")
             shells = []
             if env:
-                shells += env.strip().lower().split(',')
+                shells += env.strip().lower().split(",")
             elif cfg.has_option(section, option):
                 shells += [cfg.get(section, option).strip().lower()]
             else:  # try all by default
                 shells += DEFAULT_PYTHON_SHELLS.keys()
             # always add standard shell as fallback
-            shells += ['python']
-            start_python_console(self.vars, shells=shells,
-                                 banner=self.vars.pop('banner', ''))
+            shells += ["python"]
+            start_python_console(
+                self.vars, shells=shells, banner=self.vars.pop("banner", "")
+            )
 
     def _schedule(self, request, spider):
         spider = self._open_spider(request, spider)
@@ -96,19 +97,23 @@ class Shell:
 
     def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
         from twisted.internet import reactor
+
         if isinstance(request_or_url, Request):
             request = request_or_url
         else:
             url = any_to_uri(request_or_url)
             request = Request(url, dont_filter=True, **kwargs)
             if redirect:
-                request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
+                request.meta["handle_httpstatus_list"] = SequenceExclude(
+                    range(300, 400)
+                )
             else:
-                request.meta['handle_httpstatus_all'] = True
+                request.meta["handle_httpstatus_all"] = True
         response = None
         try:
             response, spider = threads.blockingCallFromThread(
-                reactor, self._schedule, request, spider)
+                reactor, self._schedule, request, spider
+            )
         except IgnoreRequest:
             pass
         self.populate_vars(response, request, spider)
@@ -116,20 +121,20 @@ class Shell:
     def populate_vars(self, response=None, request=None, spider=None):
         import scrapy
 
-        self.vars['scrapy'] = scrapy
-        self.vars['crawler'] = self.crawler
-        self.vars['item'] = self.item_class()
-        self.vars['settings'] = self.crawler.settings
-        self.vars['spider'] = spider
-        self.vars['request'] = request
-        self.vars['response'] = response
+        self.vars["scrapy"] = scrapy
+        self.vars["crawler"] = self.crawler
+        self.vars["item"] = self.item_class()
+        self.vars["settings"] = self.crawler.settings
+        self.vars["spider"] = spider
+        self.vars["request"] = request
+        self.vars["response"] = response
         if self.inthread:
-            self.vars['fetch'] = self.fetch
-        self.vars['view'] = open_in_browser
-        self.vars['shelp'] = self.print_help
+            self.vars["fetch"] = self.fetch
+        self.vars["view"] = open_in_browser
+        self.vars["shelp"] = self.print_help
         self.update_vars(self.vars)
         if not self.code:
-            self.vars['banner'] = self.get_help()
+            self.vars["banner"] = self.get_help()
 
     def print_help(self):
         print(self.get_help())
@@ -137,16 +142,22 @@ class Shell:
     def get_help(self):
         b = []
         b.append("Available Scrapy objects:")
-        b.append("  scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)")
+        b.append(
+            "  scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)"
+        )
         for k, v in sorted(self.vars.items()):
             if self._is_relevant(v):
                 b.append(f"  {k:<10} {v}")
         b.append("Useful shortcuts:")
         if self.inthread:
-            b.append("  fetch(url[, redirect=True]) "
-                     "Fetch URL and update local objects (by default, redirects are followed)")
-            b.append("  fetch(req)                  "
-                     "Fetch a scrapy.Request and update local objects ")
+            b.append(
+                "  fetch(url[, redirect=True]) "
+                "Fetch URL and update local objects (by default, redirects are followed)"
+            )
+            b.append(
+                "  fetch(req)                  "
+                "Fetch a scrapy.Request and update local objects "
+            )
         b.append("  shelp()           Shell help (print this help)")
         b.append("  view(response)    View response in a browser")
 
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index ac4044c64..f00447a55 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -3,7 +3,6 @@ from scrapy.utils import signal as _signal
 
 
 class SignalManager:
-
     def __init__(self, sender=dispatcher.Anonymous):
         self.sender = sender
 
@@ -21,7 +20,7 @@ class SignalManager:
         :param signal: the signal to connect to
         :type signal: object
         """
-        kwargs.setdefault('sender', self.sender)
+        kwargs.setdefault("sender", self.sender)
         return dispatcher.connect(receiver, signal, **kwargs)
 
     def disconnect(self, receiver, signal, **kwargs):
@@ -30,7 +29,7 @@ class SignalManager:
         opposite effect of the :meth:`connect` method, and the arguments
         are the same.
         """
-        kwargs.setdefault('sender', self.sender)
+        kwargs.setdefault("sender", self.sender)
         return dispatcher.disconnect(receiver, signal, **kwargs)
 
     def send_catch_log(self, signal, **kwargs):
@@ -40,7 +39,7 @@ class SignalManager:
         The keyword arguments are passed to the signal handlers (connected
         through the :meth:`connect` method).
         """
-        kwargs.setdefault('sender', self.sender)
+        kwargs.setdefault("sender", self.sender)
         return _signal.send_catch_log(signal, **kwargs)
 
     def send_catch_log_deferred(self, signal, **kwargs):
@@ -54,7 +53,7 @@ class SignalManager:
         The keyword arguments are passed to the signal handlers (connected
         through the :meth:`connect` method).
         """
-        kwargs.setdefault('sender', self.sender)
+        kwargs.setdefault("sender", self.sender)
         return _signal.send_catch_log_deferred(signal, **kwargs)
 
     def disconnect_all(self, signal, **kwargs):
@@ -64,5 +63,5 @@ class SignalManager:
         :param signal: the signal to disconnect from
         :type signal: object
         """
-        kwargs.setdefault('sender', self.sender)
+        kwargs.setdefault("sender", self.sender)
         return _signal.disconnect_all(signal, **kwargs)
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 04fda311f..02a451a2b 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -17,8 +17,8 @@ class SpiderLoader:
     """
 
     def __init__(self, settings):
-        self.spider_modules = settings.getlist('SPIDER_MODULES')
-        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
+        self.spider_modules = settings.getlist("SPIDER_MODULES")
+        self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY")
         self._spiders = {}
         self._found = defaultdict(list)
         self._load_all_spiders()
@@ -26,11 +26,13 @@ class SpiderLoader:
     def _check_name_duplicates(self):
         dupes = []
         for name, locations in self._found.items():
-            dupes.extend([
-                f"  {cls} named {name!r} (in {mod})"
-                for mod, cls in locations
-                if len(locations) > 1
-            ])
+            dupes.extend(
+                [
+                    f"  {cls} named {name!r} (in {mod})"
+                    for mod, cls in locations
+                    if len(locations) > 1
+                ]
+            )
 
         if dupes:
             dupes_string = "\n\n".join(dupes)
@@ -81,8 +83,7 @@ class SpiderLoader:
         Return the list of spider names that can handle the given request.
         """
         return [
-            name for name, cls in self._spiders.items()
-            if cls.handles_request(request)
+            name for name, cls in self._spiders.items() if cls.handles_request(request)
         ]
 
     def list(self):
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 4c923b1b3..eadc7c6ab 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -12,7 +12,6 @@ logger = logging.getLogger(__name__)
 
 
 class DepthMiddleware:
-
     def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
         self.maxdepth = maxdepth
         self.stats = stats
@@ -22,9 +21,9 @@ class DepthMiddleware:
     @classmethod
     def from_crawler(cls, crawler):
         settings = crawler.settings
-        maxdepth = settings.getint('DEPTH_LIMIT')
-        verbose = settings.getbool('DEPTH_STATS_VERBOSE')
-        prio = settings.getint('DEPTH_PRIORITY')
+        maxdepth = settings.getint("DEPTH_LIMIT")
+        verbose = settings.getbool("DEPTH_STATS_VERBOSE")
+        prio = settings.getint("DEPTH_PRIORITY")
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     def process_spider_output(self, response, result, spider):
@@ -39,28 +38,26 @@ class DepthMiddleware:
 
     def _init_depth(self, response, spider):
         # base case (depth=0)
-        if 'depth' not in response.meta:
-            response.meta['depth'] = 0
+        if "depth" not in response.meta:
+            response.meta["depth"] = 0
             if self.verbose_stats:
-                self.stats.inc_value('request_depth_count/0', spider=spider)
+                self.stats.inc_value("request_depth_count/0", spider=spider)
 
     def _filter(self, request, response, spider):
         if not isinstance(request, Request):
             return True
-        depth = response.meta['depth'] + 1
-        request.meta['depth'] = depth
+        depth = response.meta["depth"] + 1
+        request.meta["depth"] = depth
         if self.prio:
             request.priority -= depth * self.prio
         if self.maxdepth and depth > self.maxdepth:
             logger.debug(
                 "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                {'maxdepth': self.maxdepth, 'requrl': request.url},
-                extra={'spider': spider}
+                {"maxdepth": self.maxdepth, "requrl": request.url},
+                extra={"spider": spider},
             )
             return False
         if self.verbose_stats:
-            self.stats.inc_value(f'request_depth_count/{depth}',
-                                 spider=spider)
-        self.stats.max_value('request_depth_max', depth,
-                             spider=spider)
+            self.stats.inc_value(f"request_depth_count/{depth}", spider=spider)
+        self.stats.max_value("request_depth_max", depth, spider=spider)
         return True
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 9861456de..0d3e5fe0b 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -19,39 +19,41 @@ class HttpError(IgnoreRequest):
 
 
 class HttpErrorMiddleware:
-
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
 
     def __init__(self, settings):
-        self.handle_httpstatus_all = settings.getbool('HTTPERROR_ALLOW_ALL')
-        self.handle_httpstatus_list = settings.getlist('HTTPERROR_ALLOWED_CODES')
+        self.handle_httpstatus_all = settings.getbool("HTTPERROR_ALLOW_ALL")
+        self.handle_httpstatus_list = settings.getlist("HTTPERROR_ALLOWED_CODES")
 
     def process_spider_input(self, response, spider):
         if 200 <= response.status < 300:  # common case
             return
         meta = response.meta
-        if meta.get('handle_httpstatus_all', False):
+        if meta.get("handle_httpstatus_all", False):
             return
-        if 'handle_httpstatus_list' in meta:
-            allowed_statuses = meta['handle_httpstatus_list']
+        if "handle_httpstatus_list" in meta:
+            allowed_statuses = meta["handle_httpstatus_list"]
         elif self.handle_httpstatus_all:
             return
         else:
-            allowed_statuses = getattr(spider, 'handle_httpstatus_list', self.handle_httpstatus_list)
+            allowed_statuses = getattr(
+                spider, "handle_httpstatus_list", self.handle_httpstatus_list
+            )
         if response.status in allowed_statuses:
             return
-        raise HttpError(response, 'Ignoring non-200 response')
+        raise HttpError(response, "Ignoring non-200 response")
 
     def process_spider_exception(self, response, exception, spider):
         if isinstance(exception, HttpError):
-            spider.crawler.stats.inc_value('httperror/response_ignored_count')
+            spider.crawler.stats.inc_value("httperror/response_ignored_count")
             spider.crawler.stats.inc_value(
-                f'httperror/response_ignored_status_count/{response.status}'
+                f"httperror/response_ignored_status_count/{response.status}"
             )
             logger.info(
                 "Ignoring response %(response)r: HTTP status code is not handled or not allowed",
-                {'response': response}, extra={'spider': spider},
+                {"response": response},
+                extra={"spider": spider},
             )
             return []
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 98a186510..c57ec8d48 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -15,7 +15,6 @@ logger = logging.getLogger(__name__)
 
 
 class OffsiteMiddleware:
-
     def __init__(self, stats):
         self.stats = stats
 
@@ -43,22 +42,24 @@ class OffsiteMiddleware:
             self.domains_seen.add(domain)
             logger.debug(
                 "Filtered offsite request to %(domain)r: %(request)s",
-                {'domain': domain, 'request': request}, extra={'spider': spider})
-            self.stats.inc_value('offsite/domains', spider=spider)
-        self.stats.inc_value('offsite/filtered', spider=spider)
+                {"domain": domain, "request": request},
+                extra={"spider": spider},
+            )
+            self.stats.inc_value("offsite/domains", spider=spider)
+        self.stats.inc_value("offsite/filtered", spider=spider)
         return False
 
     def should_follow(self, request, spider):
         regex = self.host_regex
         # hostname can be None for wrong urls (like javascript links)
-        host = urlparse_cached(request).hostname or ''
+        host = urlparse_cached(request).hostname or ""
         return bool(regex.search(host))
 
     def get_host_regex(self, spider):
         """Override this method to implement a different offsite policy"""
-        allowed_domains = getattr(spider, 'allowed_domains', None)
+        allowed_domains = getattr(spider, "allowed_domains", None)
         if not allowed_domains:
-            return re.compile('')  # allow all by default
+            return re.compile("")  # allow all by default
         url_pattern = re.compile(r"^https?://.*$")
         port_pattern = re.compile(r":\d+$")
         domains = []
@@ -66,16 +67,20 @@ class OffsiteMiddleware:
             if domain is None:
                 continue
             if url_pattern.match(domain):
-                message = ("allowed_domains accepts only domains, not URLs. "
-                           f"Ignoring URL entry {domain} in allowed_domains.")
+                message = (
+                    "allowed_domains accepts only domains, not URLs. "
+                    f"Ignoring URL entry {domain} in allowed_domains."
+                )
                 warnings.warn(message, URLWarning)
             elif port_pattern.search(domain):
-                message = ("allowed_domains accepts only domains without ports. "
-                           f"Ignoring entry {domain} in allowed_domains.")
+                message = (
+                    "allowed_domains accepts only domains without ports. "
+                    f"Ignoring entry {domain} in allowed_domains."
+                )
                 warnings.warn(message, PortWarning)
             else:
                 domains.append(re.escape(domain))
-        regex = fr'^(.*\.)?({"|".join(domains)})$'
+        regex = rf'^(.*\.)?({"|".join(domains)})$'
         return re.compile(regex)
 
     def spider_opened(self, spider):
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 4a6c4de5e..a99b6315b 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -16,7 +16,12 @@ from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
 
 
-LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
+LOCAL_SCHEMES = (
+    "about",
+    "blob",
+    "data",
+    "filesystem",
+)
 
 POLICY_NO_REFERRER = "no-referrer"
 POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
@@ -61,11 +66,13 @@ class ReferrerPolicy:
         """
         if not url:
             return None
-        return strip_url(url,
-                         strip_credentials=True,
-                         strip_fragment=True,
-                         strip_default_port=True,
-                         origin_only=origin_only)
+        return strip_url(
+            url,
+            strip_credentials=True,
+            strip_fragment=True,
+            strip_default_port=True,
+            origin_only=origin_only,
+        )
 
     def origin(self, url):
         """Return serialized origin (scheme, host, path) for a request or response URL."""
@@ -74,12 +81,12 @@ class ReferrerPolicy:
     def potentially_trustworthy(self, url):
         # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
         parsed_url = urlparse(url)
-        if parsed_url.scheme in ('data',):
+        if parsed_url.scheme in ("data",):
             return False
         return self.tls_protected(url)
 
     def tls_protected(self, url):
-        return urlparse(url).scheme in ('https', 'ftps')
+        return urlparse(url).scheme in ("https", "ftps")
 
 
 class NoReferrerPolicy(ReferrerPolicy):
@@ -90,6 +97,7 @@ class NoReferrerPolicy(ReferrerPolicy):
     is to be sent along with requests made from a particular request client to any origin.
     The header will be omitted entirely.
     """
+
     name: str = POLICY_NO_REFERRER
 
     def referrer(self, response_url, request_url):
@@ -110,6 +118,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
 
     This is a user agent's default behavior, if no policy is otherwise specified.
     """
+
     name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
     def referrer(self, response_url, request_url):
@@ -127,6 +136,7 @@ class SameOriginPolicy(ReferrerPolicy):
     Cross-origin requests, on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
+
     name: str = POLICY_SAME_ORIGIN
 
     def referrer(self, response_url, request_url):
@@ -143,6 +153,7 @@ class OriginPolicy(ReferrerPolicy):
     when making both same-origin requests and cross-origin requests
     from a particular request client.
     """
+
     name: str = POLICY_ORIGIN
 
     def referrer(self, response_url, request_url):
@@ -162,11 +173,13 @@ class StrictOriginPolicy(ReferrerPolicy):
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
+
     name: str = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
         if (
-            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            self.tls_protected(response_url)
+            and self.potentially_trustworthy(request_url)
             or not self.tls_protected(response_url)
         ):
             return self.origin_referrer(response_url)
@@ -183,6 +196,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     is sent as referrer information when making cross-origin requests
     from a particular request client.
     """
+
     name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response_url, request_url):
@@ -209,6 +223,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
+
     name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response_url, request_url):
@@ -216,7 +231,8 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         if origin == self.origin(request_url):
             return self.stripped_referrer(response_url)
         if (
-            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            self.tls_protected(response_url)
+            and self.potentially_trustworthy(request_url)
             or not self.tls_protected(response_url)
         ):
             return self.origin_referrer(response_url)
@@ -235,6 +251,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     to insecure origins.
     Carefully consider the impact of setting such a policy for potentially sensitive documents.
     """
+
     name: str = POLICY_UNSAFE_URL
 
     def referrer(self, response_url, request_url):
@@ -247,24 +264,28 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
     with the addition that "Referer" is not sent if the parent request was
     using ``file://`` or ``s3://`` scheme.
     """
-    NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3')
+
+    NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3")
     name: str = POLICY_SCRAPY_DEFAULT
 
 
-_policy_classes = {p.name: p for p in (
-    NoReferrerPolicy,
-    NoReferrerWhenDowngradePolicy,
-    SameOriginPolicy,
-    OriginPolicy,
-    StrictOriginPolicy,
-    OriginWhenCrossOriginPolicy,
-    StrictOriginWhenCrossOriginPolicy,
-    UnsafeUrlPolicy,
-    DefaultReferrerPolicy,
-)}
+_policy_classes = {
+    p.name: p
+    for p in (
+        NoReferrerPolicy,
+        NoReferrerWhenDowngradePolicy,
+        SameOriginPolicy,
+        OriginPolicy,
+        StrictOriginPolicy,
+        OriginWhenCrossOriginPolicy,
+        StrictOriginWhenCrossOriginPolicy,
+        UnsafeUrlPolicy,
+        DefaultReferrerPolicy,
+    )
+}
 
 # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
-_policy_classes[''] = NoReferrerWhenDowngradePolicy
+_policy_classes[""] = NoReferrerWhenDowngradePolicy
 
 
 def _load_policy_class(policy, warning_only=False):
@@ -288,16 +309,14 @@ def _load_policy_class(policy, warning_only=False):
 
 
 class RefererMiddleware:
-
     def __init__(self, settings=None):
         self.default_policy = DefaultReferrerPolicy
         if settings is not None:
-            self.default_policy = _load_policy_class(
-                settings.get('REFERRER_POLICY'))
+            self.default_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('REFERER_ENABLED'):
+        if not crawler.settings.getbool("REFERER_ENABLED"):
             raise NotConfigured
         mw = cls(crawler.settings)
 
@@ -319,12 +338,12 @@ class RefererMiddleware:
           it is used if valid
         - otherwise, the policy from settings is used.
         """
-        policy_name = request.meta.get('referrer_policy')
+        policy_name = request.meta.get("referrer_policy")
         if policy_name is None:
             if isinstance(resp_or_url, Response):
-                policy_header = resp_or_url.headers.get('Referrer-Policy')
+                policy_header = resp_or_url.headers.get("Referrer-Policy")
                 if policy_header is not None:
-                    policy_name = to_unicode(policy_header.decode('latin1'))
+                    policy_name = to_unicode(policy_header.decode("latin1"))
         if policy_name is None:
             return self.default_policy()
 
@@ -342,14 +361,14 @@ class RefererMiddleware:
         if isinstance(r, Request):
             referrer = self.policy(response, r).referrer(response.url, r.url)
             if referrer is not None:
-                r.headers.setdefault('Referer', referrer)
+                r.headers.setdefault("Referer", referrer)
         return r
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
-        redirected_urls = request.meta.get('redirect_urls', [])
+        redirected_urls = request.meta.get("redirect_urls", [])
         if redirected_urls:
-            request_referrer = request.headers.get('Referer')
+            request_referrer = request.headers.get("Referer")
             # we don't patch the referrer value if there is none
             if request_referrer is not None:
                 # the request's referrer header value acts as a surrogate
@@ -359,9 +378,10 @@ class RefererMiddleware:
                 #       the information is not available using this hook
                 parent_url = safe_url_string(request_referrer)
                 policy_referrer = self.policy(parent_url, request).referrer(
-                    parent_url, request.url)
+                    parent_url, request.url
+                )
                 if policy_referrer != request_referrer:
                     if policy_referrer is None:
-                        request.headers.pop('Referer')
+                        request.headers.pop("Referer")
                     else:
-                        request.headers['Referer'] = policy_referrer
+                        request.headers["Referer"] = policy_referrer
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 7ad64d2af..9a21379f9 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -13,13 +13,12 @@ logger = logging.getLogger(__name__)
 
 
 class UrlLengthMiddleware:
-
     def __init__(self, maxlength):
         self.maxlength = maxlength
 
     @classmethod
     def from_settings(cls, settings):
-        maxlength = settings.getint('URLLENGTH_LIMIT')
+        maxlength = settings.getint("URLLENGTH_LIMIT")
         if not maxlength:
             raise NotConfigured
         return cls(maxlength)
@@ -36,9 +35,11 @@ class UrlLengthMiddleware:
         if isinstance(request, Request) and len(request.url) > self.maxlength:
             logger.info(
                 "Ignoring link (url length > %(maxlength)d): %(url)s ",
-                {'maxlength': self.maxlength, 'url': request.url},
-                extra={'spider': spider}
+                {"maxlength": self.maxlength, "url": request.url},
+                extra={"spider": spider},
+            )
+            spider.crawler.stats.inc_value(
+                "urllength/request_ignored_count", spider=spider
             )
-            spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
             return False
         return True
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index a37af97ce..f8cac5458 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -28,16 +28,16 @@ class Spider(object_ref):
     def __init__(self, name=None, **kwargs):
         if name is not None:
             self.name = name
-        elif not getattr(self, 'name', None):
+        elif not getattr(self, "name", None):
             raise ValueError(f"{type(self).__name__} must have a name")
         self.__dict__.update(kwargs)
-        if not hasattr(self, 'start_urls'):
+        if not hasattr(self, "start_urls"):
             self.start_urls = []
 
     @property
     def logger(self):
         logger = logging.getLogger(self.name)
-        return logging.LoggerAdapter(logger, {'spider': self})
+        return logging.LoggerAdapter(logger, {"spider": self})
 
     def log(self, message, level=logging.DEBUG, **kw):
         """Log the given message at the given log level
@@ -60,11 +60,12 @@ class Spider(object_ref):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        if not self.start_urls and hasattr(self, 'start_url'):
+        if not self.start_urls and hasattr(self, "start_url"):
             raise AttributeError(
                 "Crawling could not start: 'start_urls' not found "
                 "or empty (but found 'start_url' attribute instead, "
-                "did you miss an 's'?)")
+                "did you miss an 's'?)"
+            )
         for url in self.start_urls:
             yield Request(url, dont_filter=True)
 
@@ -72,11 +73,13 @@ class Spider(object_ref):
         return self.parse(response, **kwargs)
 
     def parse(self, response, **kwargs):
-        raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined')
+        raise NotImplementedError(
+            f"{self.__class__.__name__}.parse callback is not defined"
+        )
 
     @classmethod
     def update_settings(cls, settings):
-        settings.setdict(cls.custom_settings or {}, priority='spider')
+        settings.setdict(cls.custom_settings or {}, priority="spider")
 
     @classmethod
     def handles_request(cls, request):
@@ -84,7 +87,7 @@ class Spider(object_ref):
 
     @staticmethod
     def close(spider, reason):
-        closed = getattr(spider, 'closed', None)
+        closed = getattr(spider, "closed", None)
         if callable(closed):
             return closed(reason)
 
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index b514ed9ce..d75b455ae 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -34,7 +34,6 @@ _default_link_extractor = LinkExtractor()
 
 
 class Rule:
-
     def __init__(
         self,
         link_extractor=None,
@@ -95,19 +94,24 @@ class CrawlSpider(Spider):
             return
         seen = set()
         for rule_index, rule in enumerate(self._rules):
-            links = [lnk for lnk in rule.link_extractor.extract_links(response)
-                     if lnk not in seen]
+            links = [
+                lnk
+                for lnk in rule.link_extractor.extract_links(response)
+                if lnk not in seen
+            ]
             for link in rule.process_links(links):
                 seen.add(link)
                 request = self._build_request(rule_index, link)
                 yield rule.process_request(request, response)
 
     def _callback(self, response, **cb_kwargs):
-        rule = self._rules[response.meta['rule']]
-        return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow)
+        rule = self._rules[response.meta["rule"]]
+        return self._parse_response(
+            response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow
+        )
 
     def _errback(self, failure):
-        rule = self._rules[failure.request.meta['rule']]
+        rule = self._rules[failure.request.meta["rule"]]
         return self._handle_failure(failure, rule.errback)
 
     async def _parse_response(self, response, callback, cb_kwargs, follow=True):
@@ -140,5 +144,7 @@ class CrawlSpider(Spider):
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
         spider = super().from_crawler(crawler, *args, **kwargs)
-        spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
+        spider._follow_links = crawler.settings.getbool(
+            "CRAWLSPIDER_FOLLOW_LINKS", True
+        )
         return spider
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 79e12e030..b3c5ff01e 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -21,8 +21,8 @@ class XMLFeedSpider(Spider):
     use iternodes, since it's a faster and cleaner.
     """
 
-    iterator = 'iternodes'
-    itertag = 'item'
+    iterator = "iternodes"
+    itertag = "item"
     namespaces = ()
 
     def process_results(self, response, results):
@@ -44,7 +44,7 @@ class XMLFeedSpider(Spider):
 
     def parse_node(self, response, selector):
         """This method must be overridden with your custom spider functionality"""
-        if hasattr(self, 'parse_item'):  # backward compatibility
+        if hasattr(self, "parse_item"):  # backward compatibility
             return self.parse_item(response, selector)
         raise NotImplementedError
 
@@ -62,22 +62,24 @@ class XMLFeedSpider(Spider):
                 yield result_item
 
     def _parse(self, response, **kwargs):
-        if not hasattr(self, 'parse_node'):
-            raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
+        if not hasattr(self, "parse_node"):
+            raise NotConfigured(
+                "You must define parse_node method in order to scrape this XML feed"
+            )
 
         response = self.adapt_response(response)
-        if self.iterator == 'iternodes':
+        if self.iterator == "iternodes":
             nodes = self._iternodes(response)
-        elif self.iterator == 'xml':
-            selector = Selector(response, type='xml')
+        elif self.iterator == "xml":
+            selector = Selector(response, type="xml")
             self._register_namespaces(selector)
-            nodes = selector.xpath(f'//{self.itertag}')
-        elif self.iterator == 'html':
-            selector = Selector(response, type='html')
+            nodes = selector.xpath(f"//{self.itertag}")
+        elif self.iterator == "html":
+            selector = Selector(response, type="html")
             self._register_namespaces(selector)
-            nodes = selector.xpath(f'//{self.itertag}')
+            nodes = selector.xpath(f"//{self.itertag}")
         else:
-            raise NotSupported('Unsupported node iterator')
+            raise NotSupported("Unsupported node iterator")
 
         return self.parse_nodes(response, nodes)
 
@@ -100,8 +102,12 @@ class CSVFeedSpider(Spider):
     and the file's headers.
     """
 
-    delimiter = None  # When this is None, python's csv module's default delimiter is used
-    quotechar = None  # When this is None, python's csv module's default quotechar is used
+    delimiter = (
+        None  # When this is None, python's csv module's default delimiter is used
+    )
+    quotechar = (
+        None  # When this is None, python's csv module's default quotechar is used
+    )
     headers = None
 
     def process_results(self, response, results):
@@ -123,13 +129,17 @@ class CSVFeedSpider(Spider):
         process_results methods for pre and post-processing purposes.
         """
 
-        for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
+        for row in csviter(
+            response, self.delimiter, self.headers, quotechar=self.quotechar
+        ):
             ret = iterate_spider_output(self.parse_row(response, row))
             for result_item in self.process_results(response, ret):
                 yield result_item
 
     def _parse(self, response, **kwargs):
-        if not hasattr(self, 'parse_row'):
-            raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
+        if not hasattr(self, "parse_row"):
+            raise NotConfigured(
+                "You must define parse_row method in order to scrape this CSV feed"
+            )
         response = self.adapt_response(response)
         return self.parse_rows(response)
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index fe8c94e78..3cb215b0f 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -13,7 +13,7 @@ class InitSpider(Spider):
         """This method must be set as the callback of your last initialization
         request. See self.init_request() docstring for more info.
         """
-        return self.__dict__.pop('_postinit_reqs')
+        return self.__dict__.pop("_postinit_reqs")
 
     def init_request(self):
         """This function should return one initialization request, with the
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 43d138753..a1734a3b1 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -13,8 +13,8 @@ logger = logging.getLogger(__name__)
 class SitemapSpider(Spider):
 
     sitemap_urls = ()
-    sitemap_rules = [('', 'parse')]
-    sitemap_follow = ['']
+    sitemap_rules = [("", "parse")]
+    sitemap_follow = [""]
     sitemap_alternate_links = False
 
     def __init__(self, *a, **kw):
@@ -39,24 +39,27 @@ class SitemapSpider(Spider):
             yield entry
 
     def _parse_sitemap(self, response):
-        if response.url.endswith('/robots.txt'):
+        if response.url.endswith("/robots.txt"):
             for url in sitemap_urls_from_robots(response.text, base_url=response.url):
                 yield Request(url, callback=self._parse_sitemap)
         else:
             body = self._get_sitemap_body(response)
             if body is None:
-                logger.warning("Ignoring invalid sitemap: %(response)s",
-                               {'response': response}, extra={'spider': self})
+                logger.warning(
+                    "Ignoring invalid sitemap: %(response)s",
+                    {"response": response},
+                    extra={"spider": self},
+                )
                 return
 
             s = Sitemap(body)
             it = self.sitemap_filter(s)
 
-            if s.type == 'sitemapindex':
+            if s.type == "sitemapindex":
                 for loc in iterloc(it, self.sitemap_alternate_links):
                     if any(x.search(loc) for x in self._follow):
                         yield Request(loc, callback=self._parse_sitemap)
-            elif s.type == 'urlset':
+            elif s.type == "urlset":
                 for loc in iterloc(it, self.sitemap_alternate_links):
                     for r, c in self._cbs:
                         if r.search(loc):
@@ -80,7 +83,7 @@ class SitemapSpider(Spider):
         # without actually being a .xml.gz file in the first place,
         # merely XML gzip-compressed on the fly,
         # in other word, here, we have plain XML
-        if response.url.endswith('.xml') or response.url.endswith('.xml.gz'):
+        if response.url.endswith(".xml") or response.url.endswith(".xml.gz"):
             return response.body
 
 
@@ -92,8 +95,8 @@ def regex(x):
 
 def iterloc(it, alt=False):
     for d in it:
-        yield d['loc']
+        yield d["loc"]
 
         # Also consider alternate URLs (xhtml:link rel="alternate")
-        if alt and 'alternate' in d:
-            yield from d['alternate']
+        if alt and "alternate" in d:
+            yield from d["alternate"]
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 2fa84fc00..6afe0d636 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -15,9 +15,7 @@ from scrapy.utils.request import request_from_dict
 
 
 def _with_mkdir(queue_class):
-
     class DirectoriesCreated(queue_class):
-
         def __init__(self, path: Union[str, PathLike], *args, **kwargs):
             dirname = Path(path).parent
             if not dirname.exists():
@@ -28,9 +26,7 @@ def _with_mkdir(queue_class):
 
 
 def _serializable_queue(queue_class, serialize, deserialize):
-
     class SerializableQueue(queue_class):
-
         def push(self, obj):
             s = serialize(obj)
             super().push(s)
@@ -50,7 +46,9 @@ def _serializable_queue(queue_class, serialize, deserialize):
             try:
                 s = super().peek()
             except AttributeError as ex:
-                raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex
+                raise NotImplementedError(
+                    "The underlying queue class does not implement 'peek'"
+                ) from ex
             if s:
                 return deserialize(s)
 
@@ -58,9 +56,7 @@ def _serializable_queue(queue_class, serialize, deserialize):
 
 
 def _scrapy_serialization_queue(queue_class):
-
     class ScrapyRequestQueue(queue_class):
-
         def __init__(self, crawler, key):
             self.spider = crawler.spider
             super().__init__(key)
@@ -95,7 +91,6 @@ def _scrapy_serialization_queue(queue_class):
 
 
 def _scrapy_non_serialization_queue(queue_class):
-
     class ScrapyRequestQueue(queue_class):
         @classmethod
         def from_crawler(cls, crawler, *args, **kwargs):
@@ -111,7 +106,9 @@ def _scrapy_non_serialization_queue(queue_class):
             try:
                 s = super().peek()
             except AttributeError as ex:
-                raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex
+                raise NotImplementedError(
+                    "The underlying queue class does not implement 'peek'"
+                ) from ex
             return s
 
     return ScrapyRequestQueue
@@ -127,24 +124,16 @@ def _pickle_serialize(obj):
 
 
 _PickleFifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.FifoDiskQueue),
-    _pickle_serialize,
-    pickle.loads
+    _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads
 )
 _PickleLifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.LifoDiskQueue),
-    _pickle_serialize,
-    pickle.loads
+    _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads
 )
 _MarshalFifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.FifoDiskQueue),
-    marshal.dumps,
-    marshal.loads
+    _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads
 )
 _MarshalLifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.LifoDiskQueue),
-    marshal.dumps,
-    marshal.loads
+    _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads
 )
 
 # public queue classes
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index ba7d1a6bf..4181c7a2f 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -8,9 +8,8 @@ logger = logging.getLogger(__name__)
 
 
 class StatsCollector:
-
     def __init__(self, crawler):
-        self._dump = crawler.settings.getbool('STATS_DUMP')
+        self._dump = crawler.settings.getbool("STATS_DUMP")
         self._stats = {}
 
     def get_value(self, key, default=None, spider=None):
@@ -43,8 +42,10 @@ class StatsCollector:
 
     def close_spider(self, spider, reason):
         if self._dump:
-            logger.info("Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
-                        extra={'spider': spider})
+            logger.info(
+                "Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
+                extra={"spider": spider},
+            )
         self._persist_stats(self._stats, spider)
 
     def _persist_stats(self, stats, spider):
@@ -52,7 +53,6 @@ class StatsCollector:
 
 
 class MemoryStatsCollector(StatsCollector):
-
     def __init__(self, crawler):
         super().__init__(crawler)
         self.spider_stats = {}
@@ -62,7 +62,6 @@ class MemoryStatsCollector(StatsCollector):
 
 
 class DummyStatsCollector(StatsCollector):
-
     def get_value(self, key, default=None, spider=None):
         return default
 
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index c84b51e8c..0505db343 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -9,7 +9,7 @@ async def collect_asyncgen(result: AsyncIterable) -> list:
 
 
 async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
-    """ Wraps an iterable (sync or async) into an async generator. """
+    """Wraps an iterable (sync or async) into an async generator."""
     if isinstance(it, AsyncIterable):
         async for r in it:
             yield r
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 86238c4cd..32bc2e38c 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -13,26 +13,28 @@ class Root(Resource):
         return self
 
     def render(self, request):
-        total = _getarg(request, b'total', 100, int)
-        show = _getarg(request, b'show', 10, int)
+        total = _getarg(request, b"total", 100, int)
+        show = _getarg(request, b"show", 10, int)
         nlist = [random.randint(1, total) for _ in range(show)]
         request.write(b"<html><head></head><body>")
         args = request.args.copy()
         for nl in nlist:
-            args['n'] = nl
+            args["n"] = nl
             argstr = urlencode(args, doseq=True)
-            request.write(f"<a href='/follow?{argstr}'>follow {nl}</a><br>"
-                          .encode('utf8'))
+            request.write(
+                f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode("utf8")
+            )
         request.write(b"</body></html>")
-        return b''
+        return b""
 
 
 def _getarg(request, name, default=None, type=str):
     return type(request.args[name][0]) if name in request.args else default
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     from twisted.internet import reactor
+
     root = Root()
     factory = Site(root)
     httpPort = reactor.listenTCP(8998, Site(root))
@@ -40,5 +42,6 @@ if __name__ == '__main__':
     def _print_listening():
         httpHost = httpPort.getHost()
         print(f"Bench server at http://{httpHost.host}:{httpHost.port}")
+
     reactor.callWhenRunning(_print_listening)
     reactor.run()
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 39a681001..085ee7d25 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -4,6 +4,7 @@
 def is_botocore_available():
     try:
         import botocore  # noqa: F401
+
         return True
     except ImportError:
         return False
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 8ea42ce75..2f1569ab6 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -19,8 +19,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
 
     def _check_components(complist):
         if len({convert(c) for c in complist}) != len(complist):
-            raise ValueError(f'Some paths in {complist!r} convert to the same object, '
-                             'please update your settings')
+            raise ValueError(
+                f"Some paths in {complist!r} convert to the same object, "
+                "please update your settings"
+            )
 
     def _map_keys(compdict):
         if isinstance(compdict, BaseSettings):
@@ -28,10 +30,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
             for k, v in compdict.items():
                 prio = compdict.getpriority(k)
                 if compbs.getpriority(convert(k)) == prio:
-                    raise ValueError(f'Some paths in {list(compdict.keys())!r} '
-                                     'convert to the same '
-                                     'object, please update your settings'
-                                     )
+                    raise ValueError(
+                        f"Some paths in {list(compdict.keys())!r} "
+                        "convert to the same "
+                        "object, please update your settings"
+                    )
                 else:
                     compbs.set(convert(k), v, priority=prio)
             return compbs
@@ -42,8 +45,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         """Fail if a value in the components dict is not a real number or None."""
         for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
-                raise ValueError(f'Invalid value {value} for component {name}, '
-                                 'please provide a real number or None instead')
+                raise ValueError(
+                    f"Invalid value {value} for component {name}, "
+                    "please provide a real number or None instead"
+                )
 
     if isinstance(custom, (list, tuple)):
         _check_components(custom)
@@ -61,30 +66,33 @@ def arglist_to_dict(arglist):
     """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
     dict
     """
-    return dict(x.split('=', 1) for x in arglist)
+    return dict(x.split("=", 1) for x in arglist)
 
 
-def closest_scrapy_cfg(path: Union[str, os.PathLike] = '.', prevpath: Optional[Union[str, os.PathLike]] = None) -> str:
+def closest_scrapy_cfg(
+    path: Union[str, os.PathLike] = ".",
+    prevpath: Optional[Union[str, os.PathLike]] = None,
+) -> str:
     """Return the path to the closest scrapy.cfg file by traversing the current
     directory and its parents
     """
     if prevpath is not None and str(path) == str(prevpath):
-        return ''
+        return ""
     path = Path(path).resolve()
-    cfgfile = path / 'scrapy.cfg'
+    cfgfile = path / "scrapy.cfg"
     if cfgfile.exists():
         return str(cfgfile)
     return closest_scrapy_cfg(path.parent, path)
 
 
-def init_env(project='default', set_syspath=True):
+def init_env(project="default", set_syspath=True):
     """Initialize environment to use command-line tool from inside a project
     dir. This sets the Scrapy settings module and modifies the Python path to
     be able to locate the project module.
     """
     cfg = get_config()
-    if cfg.has_option('settings', project):
-        os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)
+    if cfg.has_option("settings", project):
+        os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project)
     closest = closest_scrapy_cfg()
     if closest:
         projdir = str(Path(closest).parent)
@@ -101,12 +109,14 @@ def get_config(use_closest=True):
 
 
 def get_sources(use_closest=True) -> List[str]:
-    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser()
+    xdg_config_home = (
+        os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser()
+    )
     sources = [
-        '/etc/scrapy.cfg',
-        r'c:\scrapy\scrapy.cfg',
-        str(Path(xdg_config_home) / 'scrapy.cfg'),
-        str(Path('~/.scrapy.cfg').expanduser()),
+        "/etc/scrapy.cfg",
+        r"c:\scrapy\scrapy.cfg",
+        str(Path(xdg_config_home) / "scrapy.cfg"),
+        str(Path("~/.scrapy.cfg").expanduser()),
     ]
     if use_closest:
         sources.append(closest_scrapy_cfg())
@@ -115,7 +125,7 @@ def get_sources(use_closest=True) -> List[str]:
 
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
-    out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
+    out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
@@ -128,15 +138,19 @@ def feed_complete_default_values_from_settings(feed, settings):
     return out
 
 
-def feed_process_params_from_cli(settings, output: List[str], output_format=None,
-                                 overwrite_output: Optional[List[str]] = None):
+def feed_process_params_from_cli(
+    settings,
+    output: List[str],
+    output_format=None,
+    overwrite_output: Optional[List[str]] = None,
+):
     """
     Receives feed export params (from the 'crawl' or 'runspider' commands),
     checks for inconsistencies in their quantities and returns a dictionary
     suitable to be used as the FEEDS setting.
     """
     valid_output_formats = without_none_values(
-        settings.getwithbase('FEED_EXPORTERS')
+        settings.getwithbase("FEED_EXPORTERS")
     ).keys()
 
     def check_valid_format(output_format):
@@ -179,28 +193,28 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None
                 "scrapy crawl quotes -O quotes.json:json"
             )
             warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
-            return {output[0]: {'format': output_format}}
+            return {output[0]: {"format": output_format}}
         raise UsageError(
-            'The -t command-line option cannot be used if multiple output '
-            'URIs are specified'
+            "The -t command-line option cannot be used if multiple output "
+            "URIs are specified"
         )
 
     result: Dict[str, Dict[str, Any]] = {}
     for element in output:
         try:
-            feed_uri, feed_format = element.rsplit(':', 1)
+            feed_uri, feed_format = element.rsplit(":", 1)
         except ValueError:
             feed_uri = element
-            feed_format = Path(element).suffix.replace('.', '')
+            feed_format = Path(element).suffix.replace(".", "")
         else:
-            if feed_uri == '-':
-                feed_uri = 'stdout:'
+            if feed_uri == "-":
+                feed_uri = "stdout:"
         check_valid_format(feed_format)
-        result[feed_uri] = {'format': feed_format}
+        result[feed_uri] = {"format": feed_format}
         if overwrite:
-            result[feed_uri]['overwrite'] = True
+            result[feed_uri]["overwrite"] = True
 
     # FEEDS setting should take precedence over the matching CLI options
-    result.update(settings.getdict('FEEDS'))
+    result.update(settings.getdict("FEEDS"))
 
     return result
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 1bc0bd45f..b64098d7b 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -2,7 +2,7 @@ from functools import wraps
 from collections import OrderedDict
 
 
-def _embed_ipython_shell(namespace={}, banner=''):
+def _embed_ipython_shell(namespace={}, banner=""):
     """Start an IPython Shell"""
     try:
         from IPython.terminal.embed import InteractiveShellEmbed
@@ -12,7 +12,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
         from IPython.frontend.terminal.ipapp import load_default_config
 
     @wraps(_embed_ipython_shell)
-    def wrapper(namespace=namespace, banner=''):
+    def wrapper(namespace=namespace, banner=""):
         config = load_default_config()
         # Always use .instance() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
@@ -20,55 +20,64 @@ def _embed_ipython_shell(namespace={}, banner=''):
         # on repeated breaks like with inspect_response()
         InteractiveShellEmbed.clear_instance()
         shell = InteractiveShellEmbed.instance(
-            banner1=banner, user_ns=namespace, config=config)
+            banner1=banner, user_ns=namespace, config=config
+        )
         shell()
+
     return wrapper
 
 
-def _embed_bpython_shell(namespace={}, banner=''):
+def _embed_bpython_shell(namespace={}, banner=""):
     """Start a bpython shell"""
     import bpython
 
     @wraps(_embed_bpython_shell)
-    def wrapper(namespace=namespace, banner=''):
+    def wrapper(namespace=namespace, banner=""):
         bpython.embed(locals_=namespace, banner=banner)
+
     return wrapper
 
 
-def _embed_ptpython_shell(namespace={}, banner=''):
+def _embed_ptpython_shell(namespace={}, banner=""):
     """Start a ptpython shell"""
     import ptpython.repl
 
     @wraps(_embed_ptpython_shell)
-    def wrapper(namespace=namespace, banner=''):
+    def wrapper(namespace=namespace, banner=""):
         print(banner)
         ptpython.repl.embed(locals=namespace)
+
     return wrapper
 
 
-def _embed_standard_shell(namespace={}, banner=''):
+def _embed_standard_shell(namespace={}, banner=""):
     """Start a standard python shell"""
     import code
+
     try:  # readline module is only available on unix systems
         import readline
     except ImportError:
         pass
     else:
         import rlcompleter  # noqa: F401
+
         readline.parse_and_bind("tab:complete")
 
     @wraps(_embed_standard_shell)
-    def wrapper(namespace=namespace, banner=''):
+    def wrapper(namespace=namespace, banner=""):
         code.interact(banner=banner, local=namespace)
+
     return wrapper
 
 
-DEFAULT_PYTHON_SHELLS = OrderedDict([
-    ('ptpython', _embed_ptpython_shell),
-    ('ipython', _embed_ipython_shell),
-    ('bpython', _embed_bpython_shell),
-    ('python', _embed_standard_shell),
-])
+DEFAULT_PYTHON_SHELLS = OrderedDict(
+    [
+        ("ptpython", _embed_ptpython_shell),
+        ("ipython", _embed_ipython_shell),
+        ("bpython", _embed_bpython_shell),
+        ("python", _embed_standard_shell),
+    ]
+)
 
 
 def get_shell_embed_func(shells=None, known_shells=None):
@@ -89,7 +98,7 @@ def get_shell_embed_func(shells=None, known_shells=None):
                 continue
 
 
-def start_python_console(namespace=None, banner='', shells=None):
+def start_python_console(namespace=None, banner="", shells=None):
     """Start Python console bound to the given namespace.
     Readline support and tab completion will be used on Unix, if available.
     """
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 74f82ad75..3175e5fdc 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -9,52 +9,54 @@ from w3lib.http import basic_auth_header
 
 class CurlParser(argparse.ArgumentParser):
     def error(self, message):
-        error_msg = f'There was an error parsing the curl command: {message}'
+        error_msg = f"There was an error parsing the curl command: {message}"
         raise ValueError(error_msg)
 
 
 curl_parser = CurlParser()
-curl_parser.add_argument('url')
-curl_parser.add_argument('-H', '--header', dest='headers', action='append')
-curl_parser.add_argument('-X', '--request', dest='method')
-curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
-curl_parser.add_argument('-u', '--user', dest='auth')
+curl_parser.add_argument("url")
+curl_parser.add_argument("-H", "--header", dest="headers", action="append")
+curl_parser.add_argument("-X", "--request", dest="method")
+curl_parser.add_argument("-d", "--data", "--data-raw", dest="data")
+curl_parser.add_argument("-u", "--user", dest="auth")
 
 
 safe_to_ignore_arguments = [
-    ['--compressed'],
+    ["--compressed"],
     # `--compressed` argument is not safe to ignore, but it's included here
     # because the `HttpCompressionMiddleware` is enabled by default
-    ['-s', '--silent'],
-    ['-v', '--verbose'],
-    ['-#', '--progress-bar']
+    ["-s", "--silent"],
+    ["-v", "--verbose"],
+    ["-#", "--progress-bar"],
 ]
 
 for argument in safe_to_ignore_arguments:
-    curl_parser.add_argument(*argument, action='store_true')
+    curl_parser.add_argument(*argument, action="store_true")
 
 
 def _parse_headers_and_cookies(parsed_args):
     headers = []
     cookies = {}
     for header in parsed_args.headers or ():
-        name, val = header.split(':', 1)
+        name, val = header.split(":", 1)
         name = name.strip()
         val = val.strip()
-        if name.title() == 'Cookie':
+        if name.title() == "Cookie":
             for name, morsel in SimpleCookie(val).items():
                 cookies[name] = morsel.value
         else:
             headers.append((name, val))
 
     if parsed_args.auth:
-        user, password = parsed_args.auth.split(':', 1)
-        headers.append(('Authorization', basic_auth_header(user, password)))
+        user, password = parsed_args.auth.split(":", 1)
+        headers.append(("Authorization", basic_auth_header(user, password)))
 
     return headers, cookies
 
 
-def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict:
+def curl_to_request_kwargs(
+    curl_command: str, ignore_unknown_options: bool = True
+) -> dict:
     """Convert a cURL command syntax to Request kwargs.
 
     :param str curl_command: string containing the curl command
@@ -66,7 +68,7 @@ def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = Tru
 
     curl_args = split(curl_command)
 
-    if curl_args[0] != 'curl':
+    if curl_args[0] != "curl":
         raise ValueError('A curl command must start with "curl"')
 
     parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
@@ -84,23 +86,23 @@ def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = Tru
     # needs the scheme to work
     parsed_url = urlparse(url)
     if not parsed_url.scheme:
-        url = 'http://' + url
+        url = "http://" + url
 
-    method = parsed_args.method or 'GET'
+    method = parsed_args.method or "GET"
 
-    result = {'method': method.upper(), 'url': url}
+    result = {"method": method.upper(), "url": url}
 
     headers, cookies = _parse_headers_and_cookies(parsed_args)
 
     if headers:
-        result['headers'] = headers
+        result["headers"] = headers
     if cookies:
-        result['cookies'] = cookies
+        result["cookies"] = cookies
     if parsed_args.data:
-        result['body'] = parsed_args.data
+        result["body"] = parsed_args.data
         if not parsed_args.method:
             # if the "data" is specified but the "method" is not specified,
             # the default method is 'POST'
-            result['method'] = 'POST'
+            result["method"] = "POST"
 
     return result
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 47df8a717..10497e9ed 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -30,10 +30,12 @@ class CaselessDict(dict):
 
     def __contains__(self, key):
         return dict.__contains__(self, self.normkey(key))
+
     has_key = __contains__
 
     def __copy__(self):
         return self.__class__(self)
+
     copy = __copy__
 
     def normkey(self, key):
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index fef3882cb..4e684645b 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -19,6 +19,7 @@ def deprecated(use_instead=None):
                 message += f" Use {use_instead} instead."
             warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
             return func(*args, **kwargs)
+
         return wrapped
 
     if callable(use_instead):
@@ -29,9 +30,11 @@ def deprecated(use_instead=None):
 
 def defers(func):
     """Decorator to make sure a function always returns a deferred"""
+
     @wraps(func)
     def wrapped(*a, **kw):
         return defer.maybeDeferred(func, *a, **kw)
+
     return wrapped
 
 
@@ -39,7 +42,9 @@ def inthread(func):
     """Decorator to call a function in a thread and return a deferred with the
     result
     """
+
     @wraps(func)
     def wrapped(*a, **kw):
         return threads.deferToThread(func, *a, **kw)
+
     return wrapped
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index ddacfaa49..4af2a647a 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -16,7 +16,7 @@ from typing import (
     Iterator,
     List,
     Optional,
-    Union
+    Union,
 )
 
 from twisted.internet import defer
@@ -26,7 +26,10 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
+from scrapy.utils.reactor import (
+    is_asyncio_reactor_installed,
+    get_asyncio_event_loop_policy,
+)
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -37,6 +40,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     before attending pending delayed calls, so do not set delay to zero.
     """
     from twisted.internet import reactor
+
     d = Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
@@ -50,6 +54,7 @@ def defer_succeed(result) -> Deferred:
     before attending pending delayed calls, so do not set delay to zero.
     """
     from twisted.internet import reactor
+
     d = Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d
@@ -80,7 +85,9 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
         return defer_result(result)
 
 
-def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList:
+def parallel(
+    iterable: Iterable, count: int, callable: Callable, *args, **named
+) -> DeferredList:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
@@ -92,7 +99,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)
 
 
 class _AsyncCooperatorAdapter(Iterator):
-    """ A class that wraps an async iterable into a normal iterator suitable
+    """A class that wraps an async iterable into a normal iterator suitable
     for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
     it calls the callable directly in the callback, instead of providing a more
     generic interface.
@@ -136,7 +143,14 @@ class _AsyncCooperatorAdapter(Iterator):
     Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
     goal.
     """
-    def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs):
+
+    def __init__(
+        self,
+        aiterable: AsyncIterable,
+        callable: Callable,
+        *callable_args,
+        **callable_kwargs
+    ):
         self.aiterator = aiterable.__aiter__()
         self.callable = callable
         self.callable_args = callable_args
@@ -186,8 +200,10 @@ class _AsyncCooperatorAdapter(Iterator):
         return d
 
 
-def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList:
-    """ Like parallel but for async iterators """
+def parallel_async(
+    async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named
+) -> DeferredList:
+    """Like parallel but for async iterators"""
     coop = Cooperator()
     work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
     dl = DeferredList([coop.coiterate(work) for _ in range(count)])
@@ -203,14 +219,19 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     return d
 
 
-def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
+def process_chain_both(
+    callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
+) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d = Deferred()
     for cb, eb in zip(callbacks, errbacks):
         d.addCallbacks(
-            callback=cb, errback=eb,
-            callbackArgs=a, callbackKeywords=kw,
-            errbackArgs=a, errbackKeywords=kw,
+            callback=cb,
+            errback=eb,
+            callbackArgs=a,
+            callbackKeywords=kw,
+            errbackArgs=a,
+            errbackKeywords=kw,
         )
     if isinstance(input, failure.Failure):
         d.errback(input)
@@ -243,7 +264,9 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
             errback(failure.Failure(), *a, **kw)
 
 
-async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator:
+async def aiter_errback(
+    aiterable: AsyncIterable, errback: Callable, *a, **kw
+) -> AsyncGenerator:
     """Wraps an async iterable calling an errback if an error is caught while
     iterating it. Similar to scrapy.utils.defer.iter_errback()
     """
@@ -273,19 +296,21 @@ def deferred_from_coro(o) -> Any:
 
 
 def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
-    """ Converts a coroutine function into a function that returns a Deferred.
+    """Converts a coroutine function into a function that returns a Deferred.
 
     The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
     This is useful for callback chains, as callback functions are called with the previous callback result.
     """
+
     @wraps(coro_f)
     def f(*coro_args, **coro_kwargs):
         return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
+
     return f
 
 
 def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
-    """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
+    """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)
     except:  # noqa: E722
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index a0c83f9f1..4757fef0a 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -6,13 +6,14 @@ from typing import List, Tuple
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def attribute(obj, oldattr, newattr, version='0.12'):
+def attribute(obj, oldattr, newattr, version="0.12"):
     cname = obj.__class__.__name__
     warnings.warn(
         f"{cname}.{oldattr} attribute is deprecated and will be no longer supported "
         f"in Scrapy {version}, use {cname}.{newattr} attribute instead",
         ScrapyDeprecationWarning,
-        stacklevel=3)
+        stacklevel=3,
+    )
 
 
 def create_deprecated_class(
@@ -24,7 +25,7 @@ def create_deprecated_class(
     old_class_path=None,
     new_class_path=None,
     subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
-    instance_warn_message="{cls} is deprecated, instantiate {new} instead."
+    instance_warn_message="{cls} is deprecated, instantiate {new} instead.",
 ):
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
@@ -67,11 +68,13 @@ def create_deprecated_class(
             old = meta.deprecated_class
             if old in bases and not (warn_once and meta.warned_on_subclass):
                 meta.warned_on_subclass = True
-                msg = subclass_warn_message.format(cls=_clspath(cls),
-                                                   old=_clspath(old, old_class_path),
-                                                   new=_clspath(new_class, new_class_path))
+                msg = subclass_warn_message.format(
+                    cls=_clspath(cls),
+                    old=_clspath(old, old_class_path),
+                    new=_clspath(new_class, new_class_path),
+                )
                 if warn_once:
-                    msg += ' (warning only on first subclass, there may be others)'
+                    msg += " (warning only on first subclass, there may be others)"
                 warnings.warn(msg, warn_category, stacklevel=2)
             super().__init__(name, bases, clsdict_)
 
@@ -79,8 +82,7 @@ def create_deprecated_class(
         # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
         # for implementation details
         def __instancecheck__(cls, inst):
-            return any(cls.__subclasscheck__(c)
-                       for c in (type(inst), inst.__class__))
+            return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__))
 
         def __subclasscheck__(cls, sub):
             if cls is not DeprecatedClass.deprecated_class:
@@ -93,14 +95,16 @@ def create_deprecated_class(
             if not inspect.isclass(sub):
                 raise TypeError("issubclass() arg 1 must be a class")
 
-            mro = getattr(sub, '__mro__', ())
+            mro = getattr(sub, "__mro__", ())
             return any(c in {cls, new_class} for c in mro)
 
         def __call__(cls, *args, **kwargs):
             old = DeprecatedClass.deprecated_class
             if cls is old:
-                msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
-                                                   new=_clspath(new_class, new_class_path))
+                msg = instance_warn_message.format(
+                    cls=_clspath(cls, old_class_path),
+                    new=_clspath(new_class, new_class_path),
+                )
                 warnings.warn(msg, warn_category, stacklevel=2)
             return super().__call__(*args, **kwargs)
 
@@ -124,7 +128,7 @@ def create_deprecated_class(
 def _clspath(cls, forced=None):
     if forced is not None:
         return forced
-    return f'{cls.__module__}.{cls.__name__}'
+    return f"{cls.__module__}.{cls.__name__}"
 
 
 DEPRECATION_RULES: List[Tuple[str, str]] = []
@@ -135,8 +139,10 @@ def update_classpath(path):
     for prefix, replacement in DEPRECATION_RULES:
         if isinstance(path, str) and path.startswith(prefix):
             new_path = path.replace(prefix, replacement, 1)
-            warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead",
-                          ScrapyDeprecationWarning)
+            warnings.warn(
+                f"`{path}` class is deprecated, use `{new_path}` instead",
+                ScrapyDeprecationWarning,
+            )
             return new_path
     return path
 
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 64969cb13..f6dceb87f 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -37,11 +37,12 @@ def _colorize(text, colorize=True):
     else:
         from pygments.formatters import TerminalFormatter
         from pygments.lexers import PythonLexer
+
         return highlight(text, PythonLexer(), TerminalFormatter())
 
 
 def pformat(obj, *args, **kwargs):
-    return _colorize(pformat_(obj), kwargs.pop('colorize', True))
+    return _colorize(pformat_(obj), kwargs.pop("colorize", True))
 
 
 def pprint(obj, *args, **kwargs):
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 775b25ea8..9dbb4180f 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -18,8 +18,8 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
 
 
 def ftp_store_file(
-        *, path, file, host, port,
-        username, password, use_active_mode=False, overwrite=True):
+    *, path, file, host, port, username, password, use_active_mode=False, overwrite=True
+):
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server
     """
@@ -31,6 +31,6 @@ def ftp_store_file(
         file.seek(0)
         dirname, filename = posixpath.split(path)
         ftp_makedirs_cwd(ftp, dirname)
-        command = 'STOR' if overwrite else 'APPE'
-        ftp.storbinary(f'{command} {filename}', file)
+        command = "STOR" if overwrite else "APPE"
+        ftp.storbinary(f"{command} {filename}", file)
         file.close()
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 0810e1f1d..e5df34d2e 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -10,7 +10,7 @@ def gunzip(data):
     """
     f = GzipFile(fileobj=BytesIO(data))
     output_list = []
-    chunk = b'.'
+    chunk = b"."
     while chunk:
         try:
             chunk = f.read1(8196)
@@ -20,15 +20,15 @@ def gunzip(data):
             # see issue 87 about catching struct.error
             # some pages are quite small so output_list is empty and f.extrabuf
             # contains the whole page content
-            if output_list or getattr(f, 'extrabuf', None):
+            if output_list or getattr(f, "extrabuf", None):
                 try:
-                    output_list.append(f.extrabuf[-f.extrasize:])
+                    output_list.append(f.extrabuf[-f.extrasize :])
                 finally:
                     break
             else:
                 raise
-    return b''.join(output_list)
+    return b"".join(output_list)
 
 
 def gzip_magic_number(response):
-    return response.body[:3] == b'\x1f\x8b\x08'
+    return response.body[:3] == b"\x1f\x8b\x08"
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index a90f1d278..540035ca9 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -7,7 +7,9 @@ from weakref import WeakKeyDictionary
 from scrapy.http import Request, Response
 
 
-_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
+_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = (
+    WeakKeyDictionary()
+)
 
 
 def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index da77ca46f..7d52d35c9 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -22,60 +22,62 @@ def xmliter(obj, nodename):
     """
     nodename_patt = re.escape(nodename)
 
-    DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S)
-    HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S)
-    END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S)
-    NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S)
+    DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S)
+    HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S)
+    END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S)
+    NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S)
     text = _body_or_str(obj)
 
     document_header = re.search(DOCUMENT_HEADER_RE, text)
-    document_header = document_header.group().strip() if document_header else ''
+    document_header = document_header.group().strip() if document_header else ""
     header_end_idx = re_rsearch(HEADER_END_RE, text)
-    header_end = text[header_end_idx[1]:].strip() if header_end_idx else ''
+    header_end = text[header_end_idx[1] :].strip() if header_end_idx else ""
     namespaces = {}
     if header_end:
         for tagname in reversed(re.findall(END_TAG_RE, header_end)):
-            tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S)
+            tag = re.search(
+                rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S
+            )
             if tag:
-                namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
+                namespaces.update(
+                    reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())
+                )
 
-    r = re.compile(fr'<{nodename_patt}[\s>].*?</{nodename_patt}>', re.DOTALL)
+    r = re.compile(rf"<{nodename_patt}[\s>].*?</{nodename_patt}>", re.DOTALL)
     for match in r.finditer(text):
         nodetext = (
             document_header
             + match.group().replace(
-                nodename,
-                f'{nodename} {" ".join(namespaces.values())}',
-                1
+                nodename, f'{nodename} {" ".join(namespaces.values())}', 1
             )
             + header_end
         )
-        yield Selector(text=nodetext, type='xml')
+        yield Selector(text=nodetext, type="xml")
 
 
-def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
+def xmliter_lxml(obj, nodename, namespace=None, prefix="x"):
     from lxml import etree
+
     reader = _StreamReader(obj)
-    tag = f'{{{namespace}}}{nodename}' if namespace else nodename
+    tag = f"{{{namespace}}}{nodename}" if namespace else nodename
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
-    selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename)
+    selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
     for _, node in iterable:
-        nodetext = etree.tostring(node, encoding='unicode')
+        nodetext = etree.tostring(node, encoding="unicode")
         node.clear()
-        xs = Selector(text=nodetext, type='xml')
+        xs = Selector(text=nodetext, type="xml")
         if namespace:
             xs.register_namespace(prefix, namespace)
         yield xs.xpath(selxpath)[0]
 
 
 class _StreamReader:
-
     def __init__(self, obj):
         self._ptr = 0
         if isinstance(obj, Response):
             self._text, self.encoding = obj.body, obj.encoding
         else:
-            self._text, self.encoding = obj, 'utf-8'
+            self._text, self.encoding = obj, "utf-8"
         self._is_unicode = isinstance(self._text, str)
 
     def read(self, n=65535):
@@ -90,11 +92,11 @@ class _StreamReader:
     def _read_unicode(self, n=65535):
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return self._text[s:e].encode('utf-8')
+        return self._text[s:e].encode("utf-8")
 
 
 def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
-    """ Returns an iterator of dictionaries from the given csv object
+    """Returns an iterator of dictionaries from the given csv object
 
     obj can be:
     - a Response object
@@ -109,7 +111,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     quotechar is the character used to enclosure fields on the given obj.
     """
 
-    encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
+    encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
 
     def row_to_unicode(row_):
         return [to_unicode(field, encoding) for field in row_]
@@ -133,10 +135,15 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     for row in csv_r:
         row = row_to_unicode(row)
         if len(row) != len(headers):
-            logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, "
-                           "should be: %(csvheader)d)",
-                           {'csvlnum': csv_r.line_num, 'csvrow': len(row),
-                            'csvheader': len(headers)})
+            logger.warning(
+                "ignoring row %(csvlnum)d (length: %(csvrow)d, "
+                "should be: %(csvheader)d)",
+                {
+                    "csvlnum": csv_r.line_num,
+                    "csvrow": len(row),
+                    "csvheader": len(headers),
+                },
+            )
             continue
         yield dict(zip(headers, row))
 
@@ -153,7 +160,7 @@ def _body_or_str(obj, unicode=True):
             return obj.body
         if isinstance(obj, TextResponse):
             return obj.text
-        return obj.body.decode('utf-8')
+        return obj.body.decode("utf-8")
     if isinstance(obj, str):
-        return obj if unicode else obj.encode('utf-8')
-    return obj.decode('utf-8') if unicode else obj
+        return obj if unicode else obj.encode("utf-8")
+    return obj.decode("utf-8") if unicode else obj
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index a65f92e95..858affc03 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings
 
 
 def job_dir(settings: BaseSettings) -> Optional[str]:
-    path = settings['JOBDIR']
+    path = settings["JOBDIR"]
     if path and not Path(path).exists():
         Path(path).mkdir(parents=True)
     return path
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 78e302d19..fea4deb4b 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -37,25 +37,25 @@ class TopLevelFormatter(logging.Filter):
         self.loggers = loggers or []
 
     def filter(self, record):
-        if any(record.name.startswith(logger + '.') for logger in self.loggers):
-            record.name = record.name.split('.', 1)[0]
+        if any(record.name.startswith(logger + ".") for logger in self.loggers):
+            record.name = record.name.split(".", 1)[0]
         return True
 
 
 DEFAULT_LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'loggers': {
-        'hpack': {
-            'level': 'ERROR',
+    "version": 1,
+    "disable_existing_loggers": False,
+    "loggers": {
+        "hpack": {
+            "level": "ERROR",
         },
-        'scrapy': {
-            'level': 'DEBUG',
+        "scrapy": {
+            "level": "DEBUG",
         },
-        'twisted': {
-            'level': 'ERROR',
+        "twisted": {
+            "level": "ERROR",
         },
-    }
+    },
 }
 
 
@@ -87,7 +87,7 @@ def configure_logging(settings=None, install_root_handler=True):
         # Route warnings through python logging
         logging.captureWarnings(True)
 
-    observer = twisted_log.PythonLoggingObserver('twisted')
+    observer = twisted_log.PythonLoggingObserver("twisted")
     observer.start()
 
     dictConfig(DEFAULT_LOGGING)
@@ -95,8 +95,8 @@ def configure_logging(settings=None, install_root_handler=True):
     if isinstance(settings, dict) or settings is None:
         settings = Settings(settings)
 
-    if settings.getbool('LOG_STDOUT'):
-        sys.stdout = StreamLogger(logging.getLogger('stdout'))
+    if settings.getbool("LOG_STDOUT"):
+        sys.stdout = StreamLogger(logging.getLogger("stdout"))
 
     if install_root_handler:
         install_scrapy_root_handler(settings)
@@ -105,8 +105,10 @@ def configure_logging(settings=None, install_root_handler=True):
 def install_scrapy_root_handler(settings):
     global _scrapy_root_handler
 
-    if (_scrapy_root_handler is not None
-            and _scrapy_root_handler in logging.root.handlers):
+    if (
+        _scrapy_root_handler is not None
+        and _scrapy_root_handler in logging.root.handlers
+    ):
         logging.root.removeHandler(_scrapy_root_handler)
     logging.root.setLevel(logging.NOTSET)
     _scrapy_root_handler = _get_handler(settings)
@@ -121,43 +123,46 @@ _scrapy_root_handler = None
 
 
 def _get_handler(settings):
-    """ Return a log handler object according to settings """
-    filename = settings.get('LOG_FILE')
+    """Return a log handler object according to settings"""
+    filename = settings.get("LOG_FILE")
     if filename:
-        mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
-        encoding = settings.get('LOG_ENCODING')
+        mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
+        encoding = settings.get("LOG_ENCODING")
         handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
-    elif settings.getbool('LOG_ENABLED'):
+    elif settings.getbool("LOG_ENABLED"):
         handler = logging.StreamHandler()
     else:
         handler = logging.NullHandler()
 
     formatter = logging.Formatter(
-        fmt=settings.get('LOG_FORMAT'),
-        datefmt=settings.get('LOG_DATEFORMAT')
+        fmt=settings.get("LOG_FORMAT"), datefmt=settings.get("LOG_DATEFORMAT")
     )
     handler.setFormatter(formatter)
-    handler.setLevel(settings.get('LOG_LEVEL'))
-    if settings.getbool('LOG_SHORT_NAMES'):
-        handler.addFilter(TopLevelFormatter(['scrapy']))
+    handler.setLevel(settings.get("LOG_LEVEL"))
+    if settings.getbool("LOG_SHORT_NAMES"):
+        handler.addFilter(TopLevelFormatter(["scrapy"]))
     return handler
 
 
 def log_scrapy_info(settings: Settings) -> None:
-    logger.info("Scrapy %(version)s started (bot: %(bot)s)",
-                {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
+    logger.info(
+        "Scrapy %(version)s started (bot: %(bot)s)",
+        {"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
+    )
     versions = [
         f"{name} {version}"
         for name, version in scrapy_components_versions()
         if name != "Scrapy"
     ]
-    logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
+    logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})
 
 
 def log_reactor_info() -> None:
     from twisted.internet import reactor
+
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
     from twisted.internet import asyncioreactor
+
     if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
         logger.debug(
             "Using asyncio event loop: %s.%s",
@@ -172,10 +177,11 @@ class StreamLogger:
     Taken from:
         https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
     """
+
     def __init__(self, logger, log_level=logging.INFO):
         self.logger = logger
         self.log_level = log_level
-        self.linebuf = ''
+        self.linebuf = ""
 
     def write(self, buf):
         for line in buf.rstrip().splitlines():
@@ -194,7 +200,7 @@ class LogCounterHandler(logging.Handler):
         self.crawler = crawler
 
     def emit(self, record):
-        sname = f'log_count/{record.levelname}'
+        sname = f"log_count/{record.levelname}"
         self.crawler.stats.inc_value(sname)
 
 
@@ -204,19 +210,20 @@ def logformatter_adapter(logkws):
     and adapts it into a tuple of positional arguments for logger.log calls,
     handling backward compatibility as well.
     """
-    if not {'level', 'msg', 'args'} <= set(logkws):
-        warnings.warn('Missing keys in LogFormatter method',
-                      ScrapyDeprecationWarning)
+    if not {"level", "msg", "args"} <= set(logkws):
+        warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning)
 
-    if 'format' in logkws:
-        warnings.warn('`format` key in LogFormatter methods has been '
-                      'deprecated, use `msg` instead',
-                      ScrapyDeprecationWarning)
+    if "format" in logkws:
+        warnings.warn(
+            "`format` key in LogFormatter methods has been "
+            "deprecated, use `msg` instead",
+            ScrapyDeprecationWarning,
+        )
 
-    level = logkws.get('level', logging.INFO)
-    message = logkws.get('format', logkws.get('msg'))
+    level = logkws.get("level", logging.INFO)
+    message = logkws.get("format", logkws.get("msg"))
     # NOTE: This also handles 'args' being an empty dict, that case doesn't
     # play well in logger.log calls
-    args = logkws if not logkws.get('args') else logkws['args']
+    args = logkws if not logkws.get("args") else logkws["args"]
 
     return (level, message, args)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 1e0342ace..dfd2f767c 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -30,7 +30,7 @@ def arg_to_iter(arg):
     """
     if arg is None:
         return []
-    if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
+    if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
         return arg
     return [arg]
 
@@ -48,15 +48,16 @@ def load_object(path):
     if not isinstance(path, str):
         if callable(path):
             return path
-        raise TypeError("Unexpected argument type, expected string "
-                        f"or object, got: {type(path)}")
+        raise TypeError(
+            "Unexpected argument type, expected string " f"or object, got: {type(path)}"
+        )
 
     try:
-        dot = path.rindex('.')
+        dot = path.rindex(".")
     except ValueError:
         raise ValueError(f"Error loading object '{path}': not a full path")
 
-    module, name = path[:dot], path[dot + 1:]
+    module, name = path[:dot], path[dot + 1 :]
     mod = import_module(module)
 
     try:
@@ -78,9 +79,9 @@ def walk_modules(path):
     mods = []
     mod = import_module(path)
     mods.append(mod)
-    if hasattr(mod, '__path__'):
+    if hasattr(mod, "__path__"):
         for _, subpath, ispkg in iter_modules(mod.__path__):
-            fullpath = path + '.' + subpath
+            fullpath = path + "." + subpath
             if ispkg:
                 mods += walk_modules(fullpath)
             else:
@@ -89,7 +90,7 @@ def walk_modules(path):
     return mods
 
 
-def extract_regex(regex, text, encoding='utf-8'):
+def extract_regex(regex, text, encoding="utf-8"):
     """Extract a list of unicode strings from the given text/encoding using the following policies:
 
     * if the regex contains a named group called "extract" that will be returned
@@ -99,22 +100,23 @@ def extract_regex(regex, text, encoding='utf-8'):
     warnings.warn(
         "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
         ScrapyDeprecationWarning,
-        stacklevel=2
+        stacklevel=2,
     )
 
     if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)
 
     try:
-        strings = [regex.search(text).group('extract')]   # named group
+        strings = [regex.search(text).group("extract")]  # named group
     except Exception:
-        strings = regex.findall(text)    # full regex or numbered groups
+        strings = regex.findall(text)  # full regex or numbered groups
     strings = flatten(strings)
 
     if isinstance(text, str):
-        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
-    return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
-            for s in strings]
+        return [replace_entities(s, keep=["lt", "amp"]) for s in strings]
+    return [
+        replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings
+    ]
 
 
 def md5sum(file):
@@ -136,7 +138,7 @@ def md5sum(file):
 
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
-    return rel is not None and 'nofollow' in rel.replace(',', ' ').split()
+    return rel is not None and "nofollow" in rel.replace(",", " ").split()
 
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):
@@ -160,15 +162,15 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
         if crawler is None:
             raise ValueError("Specify at least one of settings and crawler.")
         settings = crawler.settings
-    if crawler and hasattr(objcls, 'from_crawler'):
+    if crawler and hasattr(objcls, "from_crawler"):
         instance = objcls.from_crawler(crawler, *args, **kwargs)
-        method_name = 'from_crawler'
-    elif hasattr(objcls, 'from_settings'):
+        method_name = "from_crawler"
+    elif hasattr(objcls, "from_settings"):
         instance = objcls.from_settings(settings, *args, **kwargs)
-        method_name = 'from_settings'
+        method_name = "from_settings"
     else:
         instance = objcls(*args, **kwargs)
-        method_name = '__new__'
+        method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
     return instance
@@ -221,7 +223,9 @@ def is_generator_with_return_value(callable):
 
     def returns_none(return_node):
         value = return_node.value
-        return value is None or isinstance(value, ast.NameConstant) and value.value is None
+        return (
+            value is None or isinstance(value, ast.NameConstant) and value.value is None
+        )
 
     if inspect.isgeneratorfunction(callable):
         func = callable
@@ -256,8 +260,8 @@ def warn_on_generator_with_return_value(spider, callable):
             warnings.warn(
                 f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
                 'a generator and includes a "return" statement with a value '
-                'different than None. This could lead to unexpected behaviour. Please see '
-                'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
+                "different than None. This could lead to unexpected behaviour. Please see "
+                "https://docs.python.org/3/reference/simple_stmts.html#the-return-statement "
                 'for details about the semantics of the "return" statement within generators',
                 stacklevel=2,
             )
@@ -265,9 +269,9 @@ def warn_on_generator_with_return_value(spider, callable):
         callable_name = spider.__class__.__name__ + "." + callable.__name__
         warnings.warn(
             f'Unable to determine whether or not "{callable_name}" is a generator with a return value. '
-            'This will not prevent your code from working, but it prevents Scrapy from detecting '
+            "This will not prevent your code from working, but it prevents Scrapy from detecting "
             f'potential issues in your implementation of "{callable_name}". Please, report this in the '
-            'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), '
+            "Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), "
             f'including the code of "{callable_name}"',
             stacklevel=2,
         )
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index cf867f3f8..18d856927 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -3,7 +3,7 @@ import signal
 
 signal_names = {}
 for signame in dir(signal):
-    if signame.startswith('SIG') and not signame.startswith('SIG_'):
+    if signame.startswith("SIG") and not signame.startswith("SIG_"):
         signum = getattr(signal, signame)
         if isinstance(signum, int):
             signal_names[signum] = signame
@@ -16,10 +16,11 @@ def install_shutdown_handlers(function, override_sigint=True):
     (e.g.  Pdb)
     """
     from twisted.internet import reactor
+
     reactor._handleSignals()
     signal.signal(signal.SIGTERM, function)
     if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
-    if hasattr(signal, 'SIGBREAK'):
+    if hasattr(signal, "SIGBREAK"):
         signal.signal(signal.SIGBREAK, function)
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index d21a144ad..4fbb6bcaf 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -9,23 +9,25 @@ from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 
 
-ENVVAR = 'SCRAPY_SETTINGS_MODULE'
-DATADIR_CFG_SECTION = 'datadir'
+ENVVAR = "SCRAPY_SETTINGS_MODULE"
+DATADIR_CFG_SECTION = "datadir"
 
 
 def inside_project():
-    scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
+    scrapy_module = os.environ.get("SCRAPY_SETTINGS_MODULE")
     if scrapy_module is not None:
         try:
             import_module(scrapy_module)
         except ImportError as exc:
-            warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}")
+            warnings.warn(
+                f"Cannot import scrapy settings module {scrapy_module}: {exc}"
+            )
         else:
             return True
     return bool(closest_scrapy_cfg())
 
 
-def project_data_dir(project='default') -> str:
+def project_data_dir(project="default") -> str:
     """Return the current project data dir, creating it if it doesn't exist"""
     if not inside_project():
         raise NotConfigured("Not inside a project")
@@ -35,8 +37,10 @@ def project_data_dir(project='default') -> str:
     else:
         scrapy_cfg = closest_scrapy_cfg()
         if not scrapy_cfg:
-            raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
-        d = (Path(scrapy_cfg).parent / '.scrapy').resolve()
+            raise NotConfigured(
+                "Unable to find scrapy.cfg file to infer project data dir"
+            )
+        d = (Path(scrapy_cfg).parent / ".scrapy").resolve()
     if not d.exists():
         d.mkdir(parents=True)
     return str(d)
@@ -52,7 +56,7 @@ def data_path(path: str, createdir=False) -> str:
         if inside_project():
             path_obj = Path(project_data_dir(), path)
         else:
-            path_obj = Path('.scrapy', path)
+            path_obj = Path(".scrapy", path)
     if createdir and not path_obj.exists():
         path_obj.mkdir(parents=True)
     return str(path_obj)
@@ -60,24 +64,27 @@ def data_path(path: str, createdir=False) -> str:
 
 def get_project_settings():
     if ENVVAR not in os.environ:
-        project = os.environ.get('SCRAPY_PROJECT', 'default')
+        project = os.environ.get("SCRAPY_PROJECT", "default")
         init_env(project)
 
     settings = Settings()
     settings_module_path = os.environ.get(ENVVAR)
     if settings_module_path:
-        settings.setmodule(settings_module_path, priority='project')
+        settings.setmodule(settings_module_path, priority="project")
 
     valid_envvars = {
-        'CHECK',
-        'PROJECT',
-        'PYTHON_SHELL',
-        'SETTINGS_MODULE',
+        "CHECK",
+        "PROJECT",
+        "PYTHON_SHELL",
+        "SETTINGS_MODULE",
     }
 
-    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
-                      k.startswith('SCRAPY_') and k.replace('SCRAPY_', '') in valid_envvars}
+    scrapy_envvars = {
+        k[7:]: v
+        for k, v in os.environ.items()
+        if k.startswith("SCRAPY_") and k.replace("SCRAPY_", "") in valid_envvars
+    }
 
-    settings.setdict(scrapy_envvars, priority='project')
+    settings.setdict(scrapy_envvars, priority="project")
 
     return settings
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9df1c91de..1771e5550 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -82,29 +82,32 @@ def unique(list_, key=lambda x: x):
     return result
 
 
-def to_unicode(text, encoding=None, errors='strict'):
+def to_unicode(text, encoding=None, errors="strict"):
     """Return the unicode representation of a bytes object ``text``. If
     ``text`` is already an unicode object, return it as-is."""
     if isinstance(text, str):
         return text
     if not isinstance(text, (bytes, str)):
-        raise TypeError('to_unicode must receive a bytes or str '
-                        f'object, got {type(text).__name__}')
+        raise TypeError(
+            "to_unicode must receive a bytes or str "
+            f"object, got {type(text).__name__}"
+        )
     if encoding is None:
-        encoding = 'utf-8'
+        encoding = "utf-8"
     return text.decode(encoding, errors)
 
 
-def to_bytes(text, encoding=None, errors='strict'):
+def to_bytes(text, encoding=None, errors="strict"):
     """Return the binary representation of ``text``. If ``text``
     is already a bytes object, return it as-is."""
     if isinstance(text, bytes):
         return text
     if not isinstance(text, str):
-        raise TypeError('to_bytes must receive a str or bytes '
-                        f'object, got {type(text).__name__}')
+        raise TypeError(
+            "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}"
+        )
     if encoding is None:
-        encoding = 'utf-8'
+        encoding = "utf-8"
     return text.encode(encoding, errors)
 
 
@@ -125,7 +128,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
     def _chunk_iter():
         offset = len(text)
         while True:
-            offset -= (chunk_size * 1024)
+            offset -= chunk_size * 1024
             if offset <= 0:
                 break
             yield (text[offset:], offset)
@@ -162,7 +165,7 @@ _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 
 def binary_is_text(data):
-    """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
+    """Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
     does not contain unprintable control characters.
     """
     if not isinstance(data, bytes):
@@ -182,16 +185,19 @@ def get_func_args(func, stripself=False):
     elif inspect.ismethoddescriptor(func):
         return []
     elif isinstance(func, partial):
-        return [x for x in get_func_args(func.func)[len(func.args):]
-                if not (func.keywords and x in func.keywords)]
-    elif hasattr(func, '__call__'):
+        return [
+            x
+            for x in get_func_args(func.func)[len(func.args) :]
+            if not (func.keywords and x in func.keywords)
+        ]
+    elif hasattr(func, "__call__"):
         if inspect.isroutine(func):
             return []
-        if getattr(func, '__name__', None) == '__call__':
+        if getattr(func, "__name__", None) == "__call__":
             return []
         return get_func_args(func.__call__, True)
     else:
-        raise TypeError(f'{type(func)} is not callable')
+        raise TypeError(f"{type(func)} is not callable")
     if stripself:
         func_args.pop(0)
     return func_args
@@ -221,10 +227,10 @@ def get_spec(func):
 
     if inspect.isfunction(func) or inspect.ismethod(func):
         spec = inspect.getfullargspec(func)
-    elif hasattr(func, '__call__'):
+    elif hasattr(func, "__call__"):
         spec = inspect.getfullargspec(func.__call__)
     else:
-        raise TypeError(f'{type(func)} is not callable')
+        raise TypeError(f"{type(func)} is not callable")
 
     defaults = spec.defaults or []
 
@@ -276,11 +282,14 @@ def global_object_name(obj):
 
 
 if hasattr(sys, "pypy_version_info"):
+
     def garbage_collect():
         # Collecting weakreferences can take two collections on PyPy.
         gc.collect()
         gc.collect()
+
 else:
+
     def garbage_collect():
         gc.collect()
 
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index ddf354d88..46d83059f 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -10,11 +10,12 @@ from scrapy.utils.misc import load_object
 def listen_tcp(portrange, host, factory):
     """Like reactor.listenTCP but tries different ports in a range."""
     from twisted.internet import reactor
+
     if len(portrange) > 2:
         raise ValueError(f"invalid portrange: {portrange}")
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
-    if not hasattr(portrange, '__iter__'):
+    if not hasattr(portrange, "__iter__"):
         return reactor.listenTCP(portrange, factory, interface=host)
     if len(portrange) == 1:
         return reactor.listenTCP(portrange[0], factory, interface=host)
@@ -39,6 +40,7 @@ class CallLaterOnce:
 
     def schedule(self, delay=0):
         from twisted.internet import reactor
+
         if self._call is None:
             self._call = reactor.callLater(delay, self)
 
@@ -93,16 +95,20 @@ def verify_installed_reactor(reactor_path):
     :mod:`~twisted.internet.reactor` does not match the specified import
     path."""
     from twisted.internet import reactor
+
     reactor_class = load_object(reactor_path)
     if not reactor.__class__ == reactor_class:
-        msg = ("The installed reactor "
-               f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
-               f"match the requested one ({reactor_path})")
+        msg = (
+            "The installed reactor "
+            f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
+            f"match the requested one ({reactor_path})"
+        )
         raise Exception(msg)
 
 
 def verify_installed_asyncio_event_loop(loop_path):
     from twisted.internet import reactor
+
     loop_class = load_object(loop_path)
     if isinstance(reactor._asyncioEventloop, loop_class):
         return
@@ -121,4 +127,5 @@ def verify_installed_asyncio_event_loop(loop_path):
 
 def is_asyncio_reactor_installed():
     from twisted.internet import reactor
+
     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index c254b9f82..c818c8700 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -7,16 +7,22 @@ from scrapy.utils.request import request_from_dict as _from_dict
 
 
 warnings.warn(
-    ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
-     " and/or scrapy.utils.request.request_from_dict instead"),
+    (
+        "Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
+        " and/or scrapy.utils.request.request_from_dict instead"
+    ),
     category=ScrapyDeprecationWarning,
     stacklevel=2,
 )
 
 
-def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict:
+def request_to_dict(
+    request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None
+) -> dict:
     return request.to_dict(spider=spider)
 
 
-def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request":
+def request_from_dict(
+    d: dict, spider: Optional["scrapy.Spider"] = None
+) -> "scrapy.Request":
     return _from_dict(d, spider=spider)
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index d1df8d335..3e29a9c57 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -69,54 +69,54 @@ def request_fingerprint(
     """
     if include_headers or keep_fragments:
         message = (
-            'Call to deprecated function '
-            'scrapy.utils.request.request_fingerprint().\n'
-            '\n'
-            'If you are using this function in a Scrapy component because you '
-            'need a non-default fingerprinting algorithm, and you are OK '
-            'with that non-default fingerprinting algorithm being used by '
-            'all Scrapy components and not just the one calling this '
-            'function, use crawler.request_fingerprinter.fingerprint() '
-            'instead in your Scrapy component (you can get the crawler '
-            'object from the \'from_crawler\' class method), and use the '
-            '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
-            'non-default fingerprinting algorithm.\n'
-            '\n'
-            'Otherwise, consider using the '
-            'scrapy.utils.request.fingerprint() function instead.\n'
-            '\n'
-            'If you switch to \'fingerprint()\', or assign the '
-            '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
-            '\'fingerprint()\', the generated fingerprints will not only be '
-            'bytes instead of a string, but they will also be different from '
-            'those generated by \'request_fingerprint()\'. Before you switch, '
-            'make sure that you understand the consequences of this (e.g. '
-            'cache invalidation) and are OK with them; otherwise, consider '
-            'implementing your own function which returns the same '
-            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+            "Call to deprecated function "
+            "scrapy.utils.request.request_fingerprint().\n"
+            "\n"
+            "If you are using this function in a Scrapy component because you "
+            "need a non-default fingerprinting algorithm, and you are OK "
+            "with that non-default fingerprinting algorithm being used by "
+            "all Scrapy components and not just the one calling this "
+            "function, use crawler.request_fingerprinter.fingerprint() "
+            "instead in your Scrapy component (you can get the crawler "
+            "object from the 'from_crawler' class method), and use the "
+            "'REQUEST_FINGERPRINTER_CLASS' setting to configure your "
+            "non-default fingerprinting algorithm.\n"
+            "\n"
+            "Otherwise, consider using the "
+            "scrapy.utils.request.fingerprint() function instead.\n"
+            "\n"
+            "If you switch to 'fingerprint()', or assign the "
+            "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses "
+            "'fingerprint()', the generated fingerprints will not only be "
+            "bytes instead of a string, but they will also be different from "
+            "those generated by 'request_fingerprint()'. Before you switch, "
+            "make sure that you understand the consequences of this (e.g. "
+            "cache invalidation) and are OK with them; otherwise, consider "
+            "implementing your own function which returns the same "
+            "fingerprints as the deprecated 'request_fingerprint()' function."
         )
     else:
         message = (
-            'Call to deprecated function '
-            'scrapy.utils.request.request_fingerprint().\n'
-            '\n'
-            'If you are using this function in a Scrapy component, and you '
-            'are OK with users of your component changing the fingerprinting '
-            'algorithm through settings, use '
-            'crawler.request_fingerprinter.fingerprint() instead in your '
-            'Scrapy component (you can get the crawler object from the '
-            '\'from_crawler\' class method).\n'
-            '\n'
-            'Otherwise, consider using the '
-            'scrapy.utils.request.fingerprint() function instead.\n'
-            '\n'
-            'Either way, the resulting fingerprints will be returned as '
-            'bytes, not as a string, and they will also be different from '
-            'those generated by \'request_fingerprint()\'. Before you switch, '
-            'make sure that you understand the consequences of this (e.g. '
-            'cache invalidation) and are OK with them; otherwise, consider '
-            'implementing your own function which returns the same '
-            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+            "Call to deprecated function "
+            "scrapy.utils.request.request_fingerprint().\n"
+            "\n"
+            "If you are using this function in a Scrapy component, and you "
+            "are OK with users of your component changing the fingerprinting "
+            "algorithm through settings, use "
+            "crawler.request_fingerprinter.fingerprint() instead in your "
+            "Scrapy component (you can get the crawler object from the "
+            "'from_crawler' class method).\n"
+            "\n"
+            "Otherwise, consider using the "
+            "scrapy.utils.request.fingerprint() function instead.\n"
+            "\n"
+            "Either way, the resulting fingerprints will be returned as "
+            "bytes, not as a string, and they will also be different from "
+            "those generated by 'request_fingerprint()'. Before you switch, "
+            "make sure that you understand the consequences of this (e.g. "
+            "cache invalidation) and are OK with them; otherwise, consider "
+            "implementing your own function which returns the same "
+            "fingerprints as the deprecated 'request_fingerprint()' function."
         )
     warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
     processed_include_headers: Optional[Tuple[bytes, ...]] = None
@@ -129,8 +129,10 @@ def request_fingerprint(
     if cache_key not in cache:
         fp = hashlib.sha1()
         fp.update(to_bytes(request.method))
-        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
-        fp.update(request.body or b'')
+        fp.update(
+            to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
+        )
+        fp.update(request.body or b"")
         if processed_include_headers:
             for part in _serialize_headers(processed_include_headers, request):
                 fp.update(part)
@@ -203,10 +205,10 @@ def fingerprint(
                         for header_value in request.headers.getlist(header)
                     ]
         fingerprint_data = {
-            'method': to_unicode(request.method),
-            'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
-            'body': (request.body or b'').hex(),
-            'headers': headers,
+            "method": to_unicode(request.method),
+            "url": canonicalize_url(request.url, keep_fragments=keep_fragments),
+            "body": (request.body or b"").hex(),
+            "headers": headers,
         }
         fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
         cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
@@ -233,35 +235,35 @@ class RequestFingerprinter:
     def __init__(self, crawler=None):
         if crawler:
             implementation = crawler.settings.get(
-                'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION"
             )
         else:
-            implementation = '2.6'
-        if implementation == '2.6':
+            implementation = "2.6"
+        if implementation == "2.6":
             message = (
-                '\'2.6\' is a deprecated value for the '
-                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
-                '\n'
-                'It is also the default value. In other words, it is normal '
-                'to get this warning if you have not defined a value for the '
-                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
-                'for backward compatibility reasons, but it will change in a '
-                'future version of Scrapy.\n'
-                '\n'
-                'See the documentation of the '
-                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
-                'information on how to handle this deprecation.'
+                "'2.6' is a deprecated value for the "
+                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n"
+                "\n"
+                "It is also the default value. In other words, it is normal "
+                "to get this warning if you have not defined a value for the "
+                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so "
+                "for backward compatibility reasons, but it will change in a "
+                "future version of Scrapy.\n"
+                "\n"
+                "See the documentation of the "
+                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for "
+                "information on how to handle this deprecation."
             )
             warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
             self._fingerprint = _request_fingerprint_as_bytes
-        elif implementation == '2.7':
+        elif implementation == "2.7":
             self._fingerprint = fingerprint
         else:
             raise ValueError(
-                f'Got an invalid value on setting '
-                f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
-                f'{implementation!r}. Valid values are \'2.6\' (deprecated) '
-                f'and \'2.7\'.'
+                f"Got an invalid value on setting "
+                f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
+                f"{implementation!r}. Valid values are '2.6' (deprecated) "
+                f"and '2.7'."
             )
 
     def fingerprint(self, request: Request):
@@ -276,7 +278,7 @@ def request_authenticate(
     """Authenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
-    request.headers['Authorization'] = basic_auth_header(username, password)
+    request.headers["Authorization"] = basic_auth_header(username, password)
 
 
 def request_httprepr(request: Request) -> bytes:
@@ -286,9 +288,9 @@ def request_httprepr(request: Request) -> bytes:
     by Twisted).
     """
     parsed = urlparse_cached(request)
-    path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
+    path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, ""))
     s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
-    s += b"Host: " + to_bytes(parsed.hostname or b'') + b"\r\n"
+    s += b"Host: " + to_bytes(parsed.hostname or b"") + b"\r\n"
     if request.headers:
         s += request.headers.to_string() + b"\r\n"
     s += b"\r\n"
@@ -297,11 +299,11 @@ def request_httprepr(request: Request) -> bytes:
 
 
 def referer_str(request: Request) -> Optional[str]:
-    """ Return Referer HTTP header suitable for logging. """
-    referrer = request.headers.get('Referer')
+    """Return Referer HTTP header suitable for logging."""
+    referrer = request.headers.get("Referer")
     if referrer is None:
         return referrer
-    return to_unicode(referrer, errors='replace')
+    return to_unicode(referrer, errors="replace")
 
 
 def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 23bd2da65..3693177e4 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -25,31 +25,35 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
-        _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding)
+        _baseurl_cache[response] = html.get_base_url(
+            text, response.url, response.encoding
+        )
     return _baseurl_cache[response]
 
 
-_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()
+_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = (
+    WeakKeyDictionary()
+)
 
 
 def get_meta_refresh(
     response: "scrapy.http.response.text.TextResponse",
-    ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
+    ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
         _metaref_cache[response] = html.get_meta_refresh(
-            text, response.url, response.encoding, ignore_tags=ignore_tags)
+            text, response.url, response.encoding, ignore_tags=ignore_tags
+        )
     return _metaref_cache[response]
 
 
 def response_status_message(status: Union[bytes, float, int, str]) -> str:
-    """Return status code plus status text descriptive message
-    """
+    """Return status code plus status text descriptive message"""
     status_int = int(status)
     message = http.RESPONSES.get(status_int, "Unknown Status")
-    return f'{status_int} {to_unicode(message)}'
+    return f"{status_int} {to_unicode(message)}"
 
 
 @deprecated
@@ -62,7 +66,7 @@ def response_httprepr(response: Response) -> bytes:
         b"HTTP/1.1 ",
         to_bytes(str(response.status)),
         b" ",
-        to_bytes(http.RESPONSES.get(response.status, b'')),
+        to_bytes(http.RESPONSES.get(response.status, b"")),
         b"\r\n",
     ]
     if response.headers:
@@ -72,26 +76,29 @@ def response_httprepr(response: Response) -> bytes:
 
 
 def open_in_browser(
-    response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"],
+    response: Union[
+        "scrapy.http.response.html.HtmlResponse",
+        "scrapy.http.response.text.TextResponse",
+    ],
     _openfunc: Callable[[str], Any] = webbrowser.open,
 ) -> Any:
     """Open the given response in a local web browser, populating the <base>
     tag for external links to work
     """
     from scrapy.http import HtmlResponse, TextResponse
+
     # XXX: this implementation is a bit dirty and could be improved
     body = response.body
     if isinstance(response, HtmlResponse):
-        if b'<base' not in body:
-            repl = fr'\1<base href="{response.url}">'
+        if b"<base" not in body:
+            repl = rf'\1<base href="{response.url}">'
             body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
             body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
-        ext = '.html'
+        ext = ".html"
     elif isinstance(response, TextResponse):
-        ext = '.txt'
+        ext = ".txt"
     else:
-        raise TypeError("Unsupported response type: "
-                        f"{response.__class__.__name__}")
+        raise TypeError("Unsupported response type: " f"{response.__class__.__name__}")
     fd, fname = tempfile.mkstemp(ext)
     os.write(fd, body)
     os.close(fd)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index fbafc9d45..b7c284174 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -5,7 +5,13 @@ import logging
 from twisted.internet.defer import DeferredList, Deferred
 from twisted.python.failure import Failure
 
-from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
+from pydispatch.dispatcher import (
+    Anonymous,
+    Any,
+    disconnect,
+    getAllReceivers,
+    liveReceivers,
+)
 from pydispatch.robustapply import robustApply
 
 from scrapy.exceptions import StopDownload
@@ -20,24 +26,36 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = named.pop('dont_log', ())
-    dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,)
-    dont_log += (StopDownload, )
-    spider = named.get('spider', None)
+    dont_log = named.pop("dont_log", ())
+    dont_log = (
+        tuple(dont_log)
+        if isinstance(dont_log, collections.abc.Sequence)
+        else (dont_log,)
+    )
+    dont_log += (StopDownload,)
+    spider = named.get("spider", None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
         try:
-            response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
+            response = robustApply(
+                receiver, signal=signal, sender=sender, *arguments, **named
+            )
             if isinstance(response, Deferred):
-                logger.error("Cannot return deferreds from signal handler: %(receiver)s",
-                             {'receiver': receiver}, extra={'spider': spider})
+                logger.error(
+                    "Cannot return deferreds from signal handler: %(receiver)s",
+                    {"receiver": receiver},
+                    extra={"spider": spider},
+                )
         except dont_log:
             result = Failure()
         except Exception:
             result = Failure()
-            logger.error("Error caught on signal handler: %(receiver)s",
-                         {'receiver': receiver},
-                         exc_info=True, extra={'spider': spider})
+            logger.error(
+                "Error caught on signal handler: %(receiver)s",
+                {"receiver": receiver},
+                exc_info=True,
+                extra={"spider": spider},
+            )
         else:
             result = response
         responses.append((receiver, result))
@@ -49,19 +67,24 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     Returns a deferred that gets fired once all signal handlers deferreds were
     fired.
     """
+
     def logerror(failure, recv):
         if dont_log is None or not isinstance(failure.value, dont_log):
-            logger.error("Error caught on signal handler: %(receiver)s",
-                         {'receiver': recv},
-                         exc_info=failure_to_exc_info(failure),
-                         extra={'spider': spider})
+            logger.error(
+                "Error caught on signal handler: %(receiver)s",
+                {"receiver": recv},
+                exc_info=failure_to_exc_info(failure),
+                extra={"spider": spider},
+            )
         return failure
 
-    dont_log = named.pop('dont_log', None)
-    spider = named.get('spider', None)
+    dont_log = named.pop("dont_log", None)
+    spider = named.get("spider", None)
     dfds = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
+        d = maybeDeferred_coro(
+            robustApply, receiver, signal=signal, sender=sender, *arguments, **named
+        )
         d.addErrback(logerror, receiver)
         d.addBoth(lambda result: (receiver, result))
         dfds.append(d)
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index a57a0c291..2622c2775 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -15,25 +15,27 @@ class Sitemap:
     (type=sitemapindex) files"""
 
     def __init__(self, xmltext):
-        xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)
+        xmlp = lxml.etree.XMLParser(
+            recover=True, remove_comments=True, resolve_entities=False
+        )
         self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
         rt = self._root.tag
-        self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt
+        self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
 
     def __iter__(self):
         for elem in self._root.getchildren():
             d = {}
             for el in elem.getchildren():
                 tag = el.tag
-                name = tag.split('}', 1)[1] if '}' in tag else tag
+                name = tag.split("}", 1)[1] if "}" in tag else tag
 
-                if name == 'link':
-                    if 'href' in el.attrib:
-                        d.setdefault('alternate', []).append(el.get('href'))
+                if name == "link":
+                    if "href" in el.attrib:
+                        d.setdefault("alternate", []).append(el.get("href"))
                 else:
-                    d[name] = el.text.strip() if el.text else ''
+                    d[name] = el.text.strip() if el.text else ""
 
-            if 'loc' in d:
+            if "loc" in d:
                 yield d
 
 
@@ -42,6 +44,6 @@ def sitemap_urls_from_robots(robots_text, base_url=None):
     robots.txt file
     """
     for line in robots_text.splitlines():
-        if line.lstrip().lower().startswith('sitemap:'):
-            url = line.split(':', 1)[1].strip()
+        if line.lstrip().lower().startswith("sitemap:"):
+            url = line.split(":", 1)[1].strip()
             yield urljoin(base_url, url)
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index b2da69404..f829bceb8 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -32,13 +32,14 @@ def iter_spider_classes(module):
             inspect.isclass(obj)
             and issubclass(obj, Spider)
             and obj.__module__ == module.__name__
-            and getattr(obj, 'name', None)
+            and getattr(obj, "name", None)
         ):
             yield obj
 
 
-def spidercls_for_request(spider_loader, request, default_spidercls=None,
-                          log_none=False, log_multiple=False):
+def spidercls_for_request(
+    spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False
+):
     """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using
@@ -54,15 +55,18 @@ def spidercls_for_request(spider_loader, request, default_spidercls=None,
         return spider_loader.load(snames[0])
 
     if len(snames) > 1 and log_multiple:
-        logger.error('More than one spider can handle: %(request)s - %(snames)s',
-                     {'request': request, 'snames': ', '.join(snames)})
+        logger.error(
+            "More than one spider can handle: %(request)s - %(snames)s",
+            {"request": request, "snames": ", ".join(snames)},
+        )
 
     if len(snames) == 0 and log_none:
-        logger.error('Unable to find spider that handles: %(request)s',
-                     {'request': request})
+        logger.error(
+            "Unable to find spider that handles: %(request)s", {"request": request}
+        )
 
     return default_spidercls
 
 
 class DefaultSpider(Spider):
-    name = 'default'
+    name = "default"
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index ea4dde882..b73261723 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -6,7 +6,7 @@ from scrapy.utils.python import to_unicode
 
 # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
 # Using the binding directly, as this code does, requires cryptography 2.4.
-SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
+SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, "SSL_OP_NO_TLSv1_3", 0)
 
 
 def ffi_buf_to_string(buf):
@@ -16,13 +16,17 @@ def ffi_buf_to_string(buf):
 def x509name_to_string(x509name):
     # from OpenSSL.crypto.X509Name.__repr__
     result_buffer = pyOpenSSLutil.ffi.new("char[]", 512)
-    pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))
+    pyOpenSSLutil.lib.X509_NAME_oneline(
+        x509name._name, result_buffer, len(result_buffer)
+    )
 
     return ffi_buf_to_string(result_buffer)
 
 
 def get_temp_key_info(ssl_object):
-    if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'):  # requires OpenSSL 1.0.2
+    if not hasattr(
+        pyOpenSSLutil.lib, "SSL_get_server_tmp_key"
+    ):  # requires OpenSSL 1.0.2
         return None
 
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
@@ -36,26 +40,28 @@ def get_temp_key_info(ssl_object):
     key_info = []
     key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
     if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
-        key_info.append('RSA')
+        key_info.append("RSA")
     elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:
-        key_info.append('DH')
+        key_info.append("DH")
     elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
-        key_info.append('ECDH')
+        key_info.append("ECDH")
         ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
         ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
-        nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key))
+        nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(
+            pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key)
+        )
         cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
         if cname == pyOpenSSLutil.ffi.NULL:
             cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
         key_info.append(ffi_buf_to_string(cname))
     else:
         key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
-    key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits')
-    return ', '.join(key_info)
+    key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")
+    return ", ".join(key_info)
 
 
 def get_openssl_version():
-    system_openssl = OpenSSL.SSL.SSLeay_version(
-        OpenSSL.SSL.SSLEAY_VERSION
-    ).decode('ascii', errors='replace')
-    return f'{OpenSSL.version.__version__} ({system_openssl})'
+    system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION).decode(
+        "ascii", errors="replace"
+    )
+    return f"{OpenSSL.version.__version__} ({system_openssl})"
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 2177817d9..89bedfc69 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -9,23 +9,23 @@ from typing import Union
 
 def render_templatefile(path: Union[str, PathLike], **kwargs):
     path_obj = Path(path)
-    raw = path_obj.read_text('utf8')
+    raw = path_obj.read_text("utf8")
 
     content = string.Template(raw).substitute(**kwargs)
 
-    render_path = path_obj.with_suffix('') if path_obj.suffix == '.tmpl' else path_obj
+    render_path = path_obj.with_suffix("") if path_obj.suffix == ".tmpl" else path_obj
 
-    if path_obj.suffix == '.tmpl':
+    if path_obj.suffix == ".tmpl":
         path_obj.rename(render_path)
 
-    render_path.write_text(content, 'utf8')
+    render_path.write_text(content, "utf8")
 
 
-CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
+CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")
 
 
 def string_camelcase(string):
-    """ Convert a word  to its CamelCase version and remove invalid chars
+    """Convert a word  to its CamelCase version and remove invalid chars
 
     >>> string_camelcase('lost-pound')
     'LostPound'
@@ -34,4 +34,4 @@ def string_camelcase(string):
     'MissingImages'
 
     """
-    return CAMELCASE_INVALID_CHARS.sub('', string.title())
+    return CAMELCASE_INVALID_CHARS.sub("", string.title())
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 9171d6224..d21065706 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -15,18 +15,19 @@ from scrapy.utils.boto import is_botocore_available
 
 
 def assert_gcs_environ():
-    if 'GCS_PROJECT_ID' not in os.environ:
+    if "GCS_PROJECT_ID" not in os.environ:
         raise SkipTest("GCS_PROJECT_ID not found")
 
 
 def skip_if_no_boto():
     if not is_botocore_available():
-        raise SkipTest('missing botocore library')
+        raise SkipTest("missing botocore library")
 
 
 def get_gcs_content_and_delete(bucket, path):
     from google.cloud import storage
-    client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
+
+    client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
     bucket = client.get_bucket(bucket)
     blob = bucket.get_blob(path)
     content = blob.download_as_string()
@@ -36,9 +37,10 @@ def get_gcs_content_and_delete(bucket, path):
 
 
 def get_ftp_content_and_delete(
-        path, host, port, username,
-        password, use_active_mode=False):
+    path, host, port, username, password, use_active_mode=False
+):
     from ftplib import FTP
+
     ftp = FTP()
     ftp.connect(host, port)
     ftp.login(username, password)
@@ -48,7 +50,8 @@ def get_ftp_content_and_delete(
 
     def buffer_data(data):
         ftp_data.append(data)
-    ftp.retrbinary(f'RETR {path}', buffer_data)
+
+    ftp.retrbinary(f"RETR {path}", buffer_data)
     dirname, filename = split(path)
     ftp.cwd(dirname)
     ftp.delete(filename)
@@ -66,7 +69,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
     # Set by default settings that prevent deprecation warnings.
     settings = {}
     if prevent_warnings:
-        settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7'
+        settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7"
     settings.update(settings_dict or {})
     runner = CrawlerRunner(settings)
     return runner.create_crawler(spidercls or Spider)
@@ -75,8 +78,8 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
 def get_pythonpath() -> str:
     """Return a PYTHONPATH suitable to use in processes so that they find this
     installation of Scrapy"""
-    scrapy_path = import_module('scrapy').__path__[0]
-    return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '')
+    scrapy_path = import_module("scrapy").__path__[0]
+    return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "")
 
 
 def get_testenv():
@@ -84,7 +87,7 @@ def get_testenv():
     this installation of Scrapy, instead of a system installed one.
     """
     env = os.environ.copy()
-    env['PYTHONPATH'] = get_pythonpath()
+    env["PYTHONPATH"] = get_pythonpath()
     return env
 
 
@@ -107,6 +110,7 @@ def mock_google_cloud_storage():
     classes and set their proper return values.
     """
     from google.cloud.storage import Client, Bucket, Blob
+
     client_mock = mock.create_autospec(Client)
 
     bucket_mock = mock.create_autospec(Bucket)
@@ -121,5 +125,6 @@ def mock_google_cloud_storage():
 def get_web_client_agent_req(url):
     from twisted.internet import reactor
     from twisted.web.client import Agent  # imports twisted.internet.reactor
+
     agent = Agent(reactor)
-    return agent.request(b'GET', url.encode('utf-8'))
+    return agent.request(b"GET", url.encode("utf-8"))
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index a54c7db95..fe5c8d74c 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -7,14 +7,15 @@ from twisted.internet import defer, protocol
 class ProcessTest:
 
     command = None
-    prefix = [sys.executable, '-m', 'scrapy.cmdline']
+    prefix = [sys.executable, "-m", "scrapy.cmdline"]
     cwd = os.getcwd()  # trial chdirs to temp dir
 
     def execute(self, args, check_code=True, settings=None):
         from twisted.internet import reactor
+
         env = os.environ.copy()
         if settings is not None:
-            env['SCRAPY_SETTINGS_MODULE'] = settings
+            env["SCRAPY_SETTINGS_MODULE"] = settings
         cmd = self.prefix + [self.command] + list(args)
         pp = TestProcessProtocol()
         pp.deferred.addBoth(self._process_finished, cmd, check_code)
@@ -32,11 +33,10 @@ class ProcessTest:
 
 
 class TestProcessProtocol(protocol.ProcessProtocol):
-
     def __init__(self):
         self.deferred = defer.Deferred()
-        self.out = b''
-        self.err = b''
+        self.out = b""
+        self.err = b""
         self.exitcode = None
 
     def outReceived(self, data):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 5d3710391..a47756c4b 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -4,9 +4,9 @@ from twisted.web import server, resource, static, util
 
 
 class SiteTest:
-
     def setUp(self):
         from twisted.internet import reactor
+
         super().setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = f"http://localhost:{self.site.getHost().port}/"
@@ -22,23 +22,34 @@ class SiteTest:
 class NoMetaRefreshRedirect(util.Redirect):
     def render(self, request):
         content = util.Redirect.render(self, request)
-        return content.replace(b'http-equiv=\"refresh\"',
-                               b'http-no-equiv=\"do-not-refresh-me\"')
+        return content.replace(
+            b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
+        )
 
 
 def test_site():
     r = resource.Resource()
     r.putChild(b"text", static.Data(b"Works", "text/plain"))
-    r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
-    r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
+    r.putChild(
+        b"html",
+        static.Data(
+            b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
+            "text/html",
+        ),
+    )
+    r.putChild(
+        b"enc-gb18030",
+        static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
+    )
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
     return server.Site(r)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     from twisted.internet import reactor
+
     port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
     print(f"http://localhost:{port.getHost().port}/")
     reactor.run()
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index b0c6a2424..9aa775a1b 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -35,8 +35,7 @@ def format_live_refs(ignore=NoneType):
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
-    for cls, wdict in sorted(live_refs.items(),
-                             key=lambda x: x[0].__name__):
+    for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__):
         if not wdict:
             continue
         if issubclass(cls, ignore):
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 21201ace5..ad3bff4f0 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -21,12 +21,14 @@ def url_is_from_any_domain(url, domains):
     if not host:
         return False
     domains = [d.lower() for d in domains]
-    return any((host == d) or (host.endswith(f'.{d}')) for d in domains)
+    return any((host == d) or (host.endswith(f".{d}")) for d in domains)
 
 
 def url_is_from_spider(url, spider):
     """Return True if the url belongs to the given spider"""
-    return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', [])))
+    return url_is_from_any_domain(
+        url, [spider.name] + list(getattr(spider, "allowed_domains", []))
+    )
 
 
 def url_has_any_extension(url, extensions):
@@ -68,9 +70,9 @@ def escape_ajax(url):
     'www.example.com/ajax.html'
     """
     defrag, frag = urldefrag(url)
-    if not frag.startswith('!'):
+    if not frag.startswith("!"):
         return url
-    return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
+    return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
 
 
 def add_http_if_no_scheme(url):
@@ -87,7 +89,7 @@ def add_http_if_no_scheme(url):
 def _is_posix_path(string):
     return bool(
         re.match(
-            r'''
+            r"""
             ^                   # start with...
             (
                 \.              # ...a single dot,
@@ -99,7 +101,7 @@ def _is_posix_path(string):
             )?      # optional match of ".", ".." or ".blabla"
             /       # at least one "/" for a file path,
             .       # and something after the "/"
-            ''',
+            """,
             string,
             flags=re.VERBOSE,
         )
@@ -109,13 +111,13 @@ def _is_posix_path(string):
 def _is_windows_path(string):
     return bool(
         re.match(
-            r'''
+            r"""
             ^
             (
                 [a-z]:\\
                 | \\\\
             )
-            ''',
+            """,
             string,
             flags=re.IGNORECASE | re.VERBOSE,
         )
@@ -134,7 +136,13 @@ def guess_scheme(url):
     return add_http_if_no_scheme(url)
 
 
-def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
+def strip_url(
+    url,
+    strip_credentials=True,
+    strip_default_port=True,
+    origin_only=False,
+    strip_fragment=True,
+):
 
     """Strip URL string from some of its components:
 
@@ -148,18 +156,24 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=
 
     parsed_url = urlparse(url)
     netloc = parsed_url.netloc
-    if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
-        netloc = netloc.split('@')[-1]
+    if (strip_credentials or origin_only) and (
+        parsed_url.username or parsed_url.password
+    ):
+        netloc = netloc.split("@")[-1]
     if strip_default_port and parsed_url.port:
-        if (parsed_url.scheme, parsed_url.port) in (('http', 80),
-                                                    ('https', 443),
-                                                    ('ftp', 21)):
-            netloc = netloc.replace(f':{parsed_url.port}', '')
-    return urlunparse((
-        parsed_url.scheme,
-        netloc,
-        '/' if origin_only else parsed_url.path,
-        '' if origin_only else parsed_url.params,
-        '' if origin_only else parsed_url.query,
-        '' if strip_fragment else parsed_url.fragment
-    ))
+        if (parsed_url.scheme, parsed_url.port) in (
+            ("http", 80),
+            ("https", 443),
+            ("ftp", 21),
+        ):
+            netloc = netloc.replace(f":{parsed_url.port}", "")
+    return urlunparse(
+        (
+            parsed_url.scheme,
+            netloc,
+            "/" if origin_only else parsed_url.path,
+            "" if origin_only else parsed_url.params,
+            "" if origin_only else parsed_url.query,
+            "" if strip_fragment else parsed_url.fragment,
+        )
+    )
diff --git a/setup.py b/setup.py
index 82ac86cdd..27581f81d 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@ from pkg_resources import parse_version
 from setuptools import setup, find_packages, __version__ as setuptools_version
 
 
-version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip()
+version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
 
 
 def has_environment_marker_platform_impl_support():
@@ -14,82 +14,82 @@ def has_environment_marker_platform_impl_support():
     it is 18.5, see:
     https://setuptools.readthedocs.io/en/latest/history.html#id235
     """
-    return parse_version(setuptools_version) >= parse_version('18.5')
+    return parse_version(setuptools_version) >= parse_version("18.5")
 
 
 install_requires = [
-    'Twisted>=18.9.0',
-    'cryptography>=3.3',
-    'cssselect>=0.9.1',
-    'itemloaders>=1.0.1',
-    'parsel>=1.5.0',
-    'pyOpenSSL>=21.0.0',
-    'queuelib>=1.4.2',
-    'service_identity>=18.1.0',
-    'w3lib>=1.17.0',
-    'zope.interface>=5.1.0',
-    'protego>=0.1.15',
-    'itemadapter>=0.1.0',
-    'setuptools',
-    'packaging',
-    'tldextract',
-    'lxml>=4.3.0',
+    "Twisted>=18.9.0",
+    "cryptography>=3.3",
+    "cssselect>=0.9.1",
+    "itemloaders>=1.0.1",
+    "parsel>=1.5.0",
+    "pyOpenSSL>=21.0.0",
+    "queuelib>=1.4.2",
+    "service_identity>=18.1.0",
+    "w3lib>=1.17.0",
+    "zope.interface>=5.1.0",
+    "protego>=0.1.15",
+    "itemadapter>=0.1.0",
+    "setuptools",
+    "packaging",
+    "tldextract",
+    "lxml>=4.3.0",
 ]
 extras_require = {}
 cpython_dependencies = [
-    'PyDispatcher>=2.0.5',
+    "PyDispatcher>=2.0.5",
 ]
 if has_environment_marker_platform_impl_support():
-    extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
+    extras_require[
+        ':platform_python_implementation == "CPython"'
+    ] = cpython_dependencies
     extras_require[':platform_python_implementation == "PyPy"'] = [
-        'PyPyDispatcher>=2.1.0',
+        "PyPyDispatcher>=2.1.0",
     ]
 else:
     install_requires.extend(cpython_dependencies)
 
 
 setup(
-    name='Scrapy',
+    name="Scrapy",
     version=version,
-    url='https://scrapy.org',
+    url="https://scrapy.org",
     project_urls={
-        'Documentation': 'https://docs.scrapy.org/',
-        'Source': 'https://github.com/scrapy/scrapy',
-        'Tracker': 'https://github.com/scrapy/scrapy/issues',
+        "Documentation": "https://docs.scrapy.org/",
+        "Source": "https://github.com/scrapy/scrapy",
+        "Tracker": "https://github.com/scrapy/scrapy/issues",
     },
-    description='A high-level Web Crawling and Web Scraping framework',
-    long_description=open('README.rst', encoding="utf-8").read(),
-    author='Scrapy developers',
-    maintainer='Pablo Hoffman',
-    maintainer_email='pablo@pablohoffman.com',
-    license='BSD',
-    packages=find_packages(exclude=('tests', 'tests.*')),
+    description="A high-level Web Crawling and Web Scraping framework",
+    long_description=open("README.rst", encoding="utf-8").read(),
+    author="Scrapy developers",
+    maintainer="Pablo Hoffman",
+    maintainer_email="pablo@pablohoffman.com",
+    license="BSD",
+    packages=find_packages(exclude=("tests", "tests.*")),
     include_package_data=True,
     zip_safe=False,
-    entry_points={
-        'console_scripts': ['scrapy = scrapy.cmdline:execute']
-    },
+    entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},
     classifiers=[
-        'Framework :: Scrapy',
-        'Development Status :: 5 - Production/Stable',
-        'Environment :: Console',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: BSD License',
-        'Operating System :: OS Independent',
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: Implementation :: CPython',
-        'Programming Language :: Python :: Implementation :: PyPy',
-        'Topic :: Internet :: WWW/HTTP',
-        'Topic :: Software Development :: Libraries :: Application Frameworks',
-        'Topic :: Software Development :: Libraries :: Python Modules',
+        "Framework :: Scrapy",
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Software Development :: Libraries :: Application Frameworks",
+        "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires='>=3.7',
+    python_requires=">=3.7",
     install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py
index 1e4ada722..5e72aa6d4 100644
--- a/tests/CrawlerProcess/asyncio_custom_loop.py
+++ b/tests/CrawlerProcess/asyncio_custom_loop.py
@@ -3,15 +3,17 @@ from scrapy.crawler import CrawlerProcess
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-    "ASYNCIO_EVENT_LOOP": "uvloop.Loop"
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+    }
+)
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index b83f6a585..1afef4d24 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -37,9 +37,11 @@ if __name__ == "__main__":
     except IndexError:
         ASYNCIO_EVENT_LOOP = None
 
-    process = CrawlerProcess(settings={
-        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-        "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,
-    })
+    process = CrawlerProcess(
+        settings={
+            "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,
+        }
+    )
     process.crawl(UrlSpider)
     process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
index d1e4a7bb5..6df6d76fa 100644
--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -3,14 +3,16 @@ from scrapy.crawler import CrawlerProcess
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+)
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index e561d63c7..01d23c963 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -2,6 +2,7 @@ import asyncio
 import sys
 
 from twisted.internet import asyncioreactor
+
 if sys.version_info >= (3, 8) and sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
@@ -11,14 +12,16 @@ from scrapy.crawler import CrawlerProcess  # noqa: E402
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+)
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
index ea8242f67..34ef00143 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
@@ -2,6 +2,7 @@ import asyncio
 import sys
 
 from twisted.internet import asyncioreactor
+
 if sys.version_info >= (3, 8) and sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
@@ -11,15 +12,17 @@ from scrapy.crawler import CrawlerProcess  # noqa: E402
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+    }
+)
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
index d24bf3031..79dd77bb2 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
@@ -4,6 +4,7 @@ import sys
 from uvloop import Loop
 
 from twisted.internet import asyncioreactor
+
 if sys.version_info >= (3, 8) and sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncio.set_event_loop(Loop())
@@ -14,15 +15,17 @@ from scrapy.crawler import CrawlerProcess  # noqa: E402
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+    }
+)
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py
index f9eab3543..7b0497bde 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver.py
@@ -8,6 +8,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
     Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution)
     """
+
     name = "caching_hostname_resolver_spider"
 
     def start_requests(self):
@@ -15,16 +16,20 @@ class CachingHostnameResolverSpider(scrapy.Spider):
 
     def parse(self, response):
         for _ in range(10):
-            yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response)
+            yield scrapy.Request(
+                response.url, dont_filter=True, callback=self.ignore_response
+            )
 
     def ignore_response(self, response):
         self.logger.info(repr(response.ip_address))
 
 
 if __name__ == "__main__":
-    process = CrawlerProcess(settings={
-        "RETRY_ENABLED": False,
-        "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
-    })
+    process = CrawlerProcess(
+        settings={
+            "RETRY_ENABLED": False,
+            "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+        }
+    )
     process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])
     process.start()
diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
index 3340d2f84..d5d19e27e 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -6,14 +6,17 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
     Finishes without a twisted.internet.error.DNSLookupError exception
     """
+
     name = "caching_hostname_resolver_spider"
     start_urls = ["http://[::1]"]
 
 
 if __name__ == "__main__":
-    process = CrawlerProcess(settings={
-        "RETRY_ENABLED": False,
-        "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
-    })
+    process = CrawlerProcess(
+        settings={
+            "RETRY_ENABLED": False,
+            "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+        }
+    )
     process.crawl(CachingHostnameResolverSpider)
     process.start()
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
index 05a98fbec..cfc4b38b7 100644
--- a/tests/CrawlerProcess/default_name_resolver.py
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -7,6 +7,7 @@ class IPv6Spider(scrapy.Spider):
     Raises a twisted.internet.error.DNSLookupError:
     the default name resolver does not handle IPv6 addresses.
     """
+
     name = "ipv6_spider"
     start_urls = ["http://[::1]"]
 
diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py
index aaa1af5c5..9f7eaf2ae 100644
--- a/tests/CrawlerProcess/multi.py
+++ b/tests/CrawlerProcess/multi.py
@@ -3,7 +3,7 @@ from scrapy.crawler import CrawlerProcess
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py
index 06b849de3..078cb72cb 100644
--- a/tests/CrawlerProcess/reactor_default.py
+++ b/tests/CrawlerProcess/reactor_default.py
@@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
index a6dff0e05..744b4ecb5 100644
--- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
@@ -5,15 +5,17 @@ from scrapy.crawler import CrawlerProcess
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py
index 907f50784..814a2a46d 100644
--- a/tests/CrawlerProcess/reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select.py
@@ -7,7 +7,7 @@ selectreactor.install()
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
index 6f48da691..37626c081 100644
--- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
@@ -13,15 +13,17 @@ installReactor(reactor)
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
index c422b13ff..b397608ec 100644
--- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
@@ -7,15 +7,17 @@ selectreactor.install()
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
index 5f6f1ae30..2d8769501 100644
--- a/tests/CrawlerProcess/simple.py
+++ b/tests/CrawlerProcess/simple.py
@@ -3,7 +3,7 @@ from scrapy.crawler import CrawlerProcess
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.py
index c6cbf949b..057d0efff 100644
--- a/tests/CrawlerProcess/twisted_reactor_asyncio.py
+++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py
@@ -3,11 +3,13 @@ from scrapy.crawler import CrawlerProcess
 
 
 class AsyncioReactorSpider(scrapy.Spider):
-    name = 'asyncio_reactor'
+    name = "asyncio_reactor"
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+)
 process.crawl(AsyncioReactorSpider)
 process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
index 56304bd23..4b6a33f66 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
@@ -3,7 +3,7 @@ from scrapy.crawler import CrawlerProcess
 
 
 class AsyncioReactorSpider(scrapy.Spider):
-    name = 'asyncio_reactor'
+    name = "asyncio_reactor"
     custom_settings = {
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
index 3f219098c..19cc08be6 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
@@ -3,14 +3,14 @@ from scrapy.crawler import CrawlerProcess
 
 
 class SelectReactorSpider(scrapy.Spider):
-    name = 'select_reactor'
+    name = "select_reactor"
     custom_settings = {
         "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
     }
 
 
 class AsyncioReactorSpider(scrapy.Spider):
-    name = 'asyncio_reactor'
+    name = "asyncio_reactor"
     custom_settings = {
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
index 72bb986bc..5c9b737c6 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -3,14 +3,14 @@ from scrapy.crawler import CrawlerProcess
 
 
 class AsyncioReactorSpider1(scrapy.Spider):
-    name = 'asyncio_reactor1'
+    name = "asyncio_reactor1"
     custom_settings = {
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
 
 
 class AsyncioReactorSpider2(scrapy.Spider):
-    name = 'asyncio_reactor2'
+    name = "asyncio_reactor2"
     custom_settings = {
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.py
index 27063260b..b2ca04672 100644
--- a/tests/CrawlerProcess/twisted_reactor_poll.py
+++ b/tests/CrawlerProcess/twisted_reactor_poll.py
@@ -3,11 +3,13 @@ from scrapy.crawler import CrawlerProcess
 
 
 class PollReactorSpider(scrapy.Spider):
-    name = 'poll_reactor'
+    name = "poll_reactor"
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+    }
+)
 process.crawl(PollReactorSpider)
 process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.py
index 9af8ceb4d..5048081f7 100644
--- a/tests/CrawlerProcess/twisted_reactor_select.py
+++ b/tests/CrawlerProcess/twisted_reactor_select.py
@@ -3,11 +3,13 @@ from scrapy.crawler import CrawlerProcess
 
 
 class SelectReactorSpider(scrapy.Spider):
-    name = 'epoll_reactor'
+    name = "epoll_reactor"
 
 
-process = CrawlerProcess(settings={
-    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
-})
+process = CrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+    }
+)
 process.crawl(SelectReactorSpider)
 process.start()
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index 417294447..26db16dd6 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -14,7 +14,7 @@ from tests.mockserver import MockServer, MockDNSServer
 # https://stackoverflow.com/a/32784190
 def createResolver(servers=None, resolvconf=None, hosts=None):
     if hosts is None:
-        hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
+        hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts"
     theResolver = Resolver(resolvconf, servers)
     hostResolver = hostsModule.Resolver(hosts)
     chain = [hostResolver, cache.CacheResolver(), theResolver]
diff --git a/tests/__init__.py b/tests/__init__.py
index be263fa16..5f0c0f7ad 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -11,24 +11,24 @@ from pathlib import Path
 # ignore system-wide proxies for tests
 # which would send requests to a totally unsuspecting server
 # (e.g. because urllib does not fully understand the proxy spec)
-os.environ['http_proxy'] = ''
-os.environ['https_proxy'] = ''
-os.environ['ftp_proxy'] = ''
+os.environ["http_proxy"] = ""
+os.environ["https_proxy"] = ""
+os.environ["ftp_proxy"] = ""
 
 # Absolutize paths to coverage config and output file because tests that
 # spawn subprocesses also changes current working directory.
 _sourceroot = Path(__file__).resolve().parent.parent
-if 'COV_CORE_CONFIG' in os.environ:
-    os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage')
-    os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG'])
+if "COV_CORE_CONFIG" in os.environ:
+    os.environ["COVERAGE_FILE"] = str(_sourceroot / ".coverage")
+    os.environ["COV_CORE_CONFIG"] = str(_sourceroot / os.environ["COV_CORE_CONFIG"])
 
-tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data')
+tests_datadir = str(Path(__file__).parent.resolve() / "sample_data")
 
 
 # In some environments accessing a non-existing host doesn't raise an
 # error. In such cases we're going to skip tests which rely on it.
 try:
-    socket.getaddrinfo('non-existing-host', 80)
+    socket.getaddrinfo("non-existing-host", 80)
     NON_EXISTING_RESOLVABLE = True
 except socket.gaierror:
     NON_EXISTING_RESOLVABLE = False
diff --git a/tests/ftpserver.py b/tests/ftpserver.py
index 6f0289e08..0c6ca3621 100644
--- a/tests/ftpserver.py
+++ b/tests/ftpserver.py
@@ -7,18 +7,18 @@ from pyftpdlib.servers import FTPServer
 
 def main():
     parser = ArgumentParser()
-    parser.add_argument('-d', '--directory')
+    parser.add_argument("-d", "--directory")
     args = parser.parse_args()
 
     authorizer = DummyAuthorizer()
-    full_permissions = 'elradfmwMT'
+    full_permissions = "elradfmwMT"
     authorizer.add_anonymous(args.directory, perm=full_permissions)
     handler = FTPHandler
     handler.authorizer = authorizer
-    address = ('127.0.0.1', 2121)
+    address = ("127.0.0.1", 2121)
     server = FTPServer(address, handler)
     server.serve_forever()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py
index 3a41b3a3e..b306437db 100644
--- a/tests/keys/__init__.py
+++ b/tests/keys/__init__.py
@@ -29,7 +29,7 @@ def generate_keys():
         key_size=2048,
         backend=default_backend(),
     )
-    (folder / 'localhost.key').write_bytes(
+    (folder / "localhost.key").write_bytes(
         key.private_bytes(
             encoding=Encoding.PEM,
             format=PrivateFormat.TraditionalOpenSSL,
@@ -58,4 +58,4 @@ def generate_keys():
         )
         .sign(key, SHA256(), default_backend())
     )
-    (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM))
+    (folder / "localhost.crt").write_bytes(cert.public_bytes(Encoding.PEM))
diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py
index 75c74daf5..e77f53666 100644
--- a/tests/mocks/dummydbm.py
+++ b/tests/mocks/dummydbm.py
@@ -4,6 +4,7 @@ import collections
 
 class DummyDB(dict):
     """Provide dummy DBM-like interface."""
+
     def close(self):
         pass
 
@@ -14,7 +15,7 @@ error = KeyError
 _DATABASES = collections.defaultdict(DummyDB)
 
 
-def open(file, flag='r', mode=0o666):
+def open(file, flag="r", mode=0o666):
     """Open or create a dummy database compatible.
 
     Arguments ``flag`` and ``mode`` are ignored.
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 6d2d95692..9f1a707c9 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -111,7 +111,6 @@ class LeafResource(resource.Resource):
 
 
 class Follow(LeafResource):
-
     def render(self, request):
         total = getarg(request, b"total", 100, type=int)
         show = getarg(request, b"show", 1, type=int)
@@ -140,13 +139,12 @@ class Follow(LeafResource):
 
 
 class Delay(LeafResource):
-
     def render_GET(self, request):
         n = getarg(request, b"n", 1, type=float)
         b = getarg(request, b"b", 1, type=int)
         if b:
             # send headers now and delay body
-            request.write('')
+            request.write("")
         self.deferRequest(request, n, self._delayedRender, request, n)
         return NOT_DONE_YET
 
@@ -156,7 +154,6 @@ class Delay(LeafResource):
 
 
 class Status(LeafResource):
-
     def render_GET(self, request):
         n = getarg(request, b"n", 200, type=int)
         request.setResponseCode(n)
@@ -164,15 +161,15 @@ class Status(LeafResource):
 
 
 class Raw(LeafResource):
-
     def render_GET(self, request):
         request.startedWriting = 1
         self.deferRequest(request, 0, self._delayedRender, request)
         return NOT_DONE_YET
+
     render_POST = render_GET
 
     def _delayedRender(self, request):
-        raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n')
+        raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")
         request.startedWriting = 1
         request.write(raw)
         request.channel.transport.loseConnection()
@@ -180,30 +177,29 @@ class Raw(LeafResource):
 
 
 class Echo(LeafResource):
-
     def render_GET(self, request):
         output = {
-            'headers': dict(
+            "headers": dict(
                 (to_unicode(k), [to_unicode(v) for v in vs])
-                for k, vs in request.requestHeaders.getAllRawHeaders()),
-            'body': to_unicode(request.content.read()),
+                for k, vs in request.requestHeaders.getAllRawHeaders()
+            ),
+            "body": to_unicode(request.content.read()),
         }
         return to_bytes(json.dumps(output))
+
     render_POST = render_GET
 
 
 class RedirectTo(LeafResource):
-
     def render(self, request):
-        goto = getarg(request, b'goto', b'/')
+        goto = getarg(request, b"goto", b"/")
         # we force the body content, otherwise Twisted redirectTo()
         # returns HTML with <meta http-equiv="refresh"
         redirectTo(goto, request)
-        return b'redirecting...'
+        return b"redirecting..."
 
 
 class Partial(LeafResource):
-
     def render_GET(self, request):
         request.setHeader(b"Content-Length", b"1024")
         self.deferRequest(request, 0, self._delayedRender, request)
@@ -215,13 +211,12 @@ class Partial(LeafResource):
 
 
 class Drop(Partial):
-
     def _delayedRender(self, request):
         abort = getarg(request, b"abort", 0, type=int)
         request.write(b"this connection will be dropped\n")
         tr = request.channel.transport
         try:
-            if abort and hasattr(tr, 'abortConnection'):
+            if abort and hasattr(tr, "abortConnection"):
                 tr.abortConnection()
             else:
                 tr.loseConnection()
@@ -230,13 +225,11 @@ class Drop(Partial):
 
 
 class ArbitraryLengthPayloadResource(LeafResource):
-
     def render(self, request):
         return request.content.read()
 
 
 class Root(resource.Resource):
-
     def __init__(self):
         resource.Resource.__init__(self)
         self.putChild(b"status", Status())
@@ -247,11 +240,15 @@ class Root(resource.Resource):
         self.putChild(b"raw", Raw())
         self.putChild(b"echo", Echo())
         self.putChild(b"payload", PayloadResource())
-        self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+        self.putChild(
+            b"xpayload",
+            resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]),
+        )
         self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
         try:
             from tests import tests_datadir
-            self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/'))))
+
+            self.putChild(b"files", File(str(Path(tests_datadir, "test_site/files/"))))
         except Exception:
             pass
         self.putChild(b"redirect-to", RedirectTo())
@@ -260,16 +257,18 @@ class Root(resource.Resource):
         return self
 
     def render(self, request):
-        return b'Scrapy mock HTTP server\n'
+        return b"Scrapy mock HTTP server\n"
 
 
 class MockServer:
-
     def __enter__(self):
-        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'],
-                          stdout=PIPE, env=get_testenv())
-        http_address = self.proc.stdout.readline().strip().decode('ascii')
-        https_address = self.proc.stdout.readline().strip().decode('ascii')
+        self.proc = Popen(
+            [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"],
+            stdout=PIPE,
+            env=get_testenv(),
+        )
+        http_address = self.proc.stdout.readline().strip().decode("ascii")
+        https_address = self.proc.stdout.readline().strip().decode("ascii")
 
         self.http_address = http_address
         self.https_address = https_address
@@ -282,7 +281,7 @@ class MockServer:
 
     def url(self, path, is_secure=False):
         host = self.https_address if is_secure else self.http_address
-        host = host.replace('0.0.0.0', '127.0.0.1')
+        host = host.replace("0.0.0.0", "127.0.0.1")
         return host + path
 
 
@@ -306,12 +305,16 @@ class MockDNSResolver:
 
 
 class MockDNSServer:
-
     def __enter__(self):
-        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
-                          stdout=PIPE, env=get_testenv())
-        self.host = '127.0.0.1'
-        self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1])
+        self.proc = Popen(
+            [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"],
+            stdout=PIPE,
+            env=get_testenv(),
+        )
+        self.host = "127.0.0.1"
+        self.port = int(
+            self.proc.stdout.readline().strip().decode("ascii").split(":")[1]
+        )
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -326,10 +329,13 @@ class MockFTPServer:
 
     def __enter__(self):
         self.path = Path(mkdtemp())
-        self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)],
-                          stderr=PIPE, env=get_testenv())
+        self.proc = Popen(
+            [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)],
+            stderr=PIPE,
+            env=get_testenv(),
+        )
         for line in self.proc.stderr:
-            if b'starting FTP server' in line:
+            if b"starting FTP server" in line:
                 break
         return self
 
@@ -339,10 +345,12 @@ class MockFTPServer:
         self.proc.communicate()
 
     def url(self, path):
-        return 'ftp://127.0.0.1:2121/' + path
+        return "ftp://127.0.0.1:2121/" + path
 
 
-def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
+def ssl_context_factory(
+    keyfile="keys/localhost.key", certfile="keys/localhost.crt", cipher_string=None
+):
     factory = ssl.DefaultOpenSSLContextFactory(
         str(Path(__file__).parent / keyfile),
         str(Path(__file__).parent / certfile),
@@ -357,7 +365,9 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http")
+    parser.add_argument(
+        "-t", "--type", type=str, choices=("http", "dns"), default="http"
+    )
     args = parser.parse_args()
 
     if args.type == "http":
@@ -370,8 +380,8 @@ if __name__ == "__main__":
         def print_listening():
             httpHost = httpPort.getHost()
             httpsHost = httpsPort.getHost()
-            httpAddress = f'http://{httpHost.host}:{httpHost.port}'
-            httpsAddress = f'https://{httpsHost.host}:{httpsHost.port}'
+            httpAddress = f"http://{httpHost.host}:{httpHost.port}"
+            httpsAddress = f"https://{httpsHost.host}:{httpsHost.port}"
             print(httpAddress)
             print(httpsAddress)
 
diff --git a/tests/pipelines.py b/tests/pipelines.py
index fed2af7d3..4e73178d5 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -4,7 +4,6 @@ Some pipelines used for testing
 
 
 class ZeroDivisionErrorPipeline:
-
     def open_spider(self, spider):
         1 / 0
 
@@ -13,6 +12,5 @@ class ZeroDivisionErrorPipeline:
 
 
 class ProcessWithZeroDivisionErrorPipiline:
-
     def process_item(self, item, spider):
         1 / 0
diff --git a/tests/spiders.py b/tests/spiders.py
index 7952e3d47..2036e8cf3 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -26,26 +26,28 @@ class MockServerSpider(Spider):
 
 class MetaSpider(MockServerSpider):
 
-    name = 'meta'
+    name = "meta"
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.meta = {}
 
     def closed(self, reason):
-        self.meta['close_reason'] = reason
+        self.meta["close_reason"] = reason
 
 
 class FollowAllSpider(MetaSpider):
 
-    name = 'follow'
+    name = "follow"
     link_extractor = LinkExtractor()
 
-    def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
+    def __init__(
+        self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs
+    ):
         super().__init__(*args, **kwargs)
         self.urls_visited = []
         self.times = []
-        qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
+        qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency}
         url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
         self.start_urls = [url]
 
@@ -58,7 +60,7 @@ class FollowAllSpider(MetaSpider):
 
 class DelaySpider(MetaSpider):
 
-    name = 'delay'
+    name = "delay"
 
     def __init__(self, n=1, b=0, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -80,7 +82,7 @@ class DelaySpider(MetaSpider):
 
 class SimpleSpider(MetaSpider):
 
-    name = 'simple'
+    name = "simple"
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -92,7 +94,7 @@ class SimpleSpider(MetaSpider):
 
 class AsyncDefSpider(SimpleSpider):
 
-    name = 'asyncdef'
+    name = "asyncdef"
 
     async def parse(self, response):
         await defer.succeed(42)
@@ -101,7 +103,7 @@ class AsyncDefSpider(SimpleSpider):
 
 class AsyncDefAsyncioSpider(SimpleSpider):
 
-    name = 'asyncdef_asyncio'
+    name = "asyncdef_asyncio"
 
     async def parse(self, response):
         await asyncio.sleep(0.2)
@@ -111,13 +113,13 @@ class AsyncDefAsyncioSpider(SimpleSpider):
 
 class AsyncDefAsyncioReturnSpider(SimpleSpider):
 
-    name = 'asyncdef_asyncio_return'
+    name = "asyncdef_asyncio_return"
 
     async def parse(self, response):
         await asyncio.sleep(0.2)
         status = await get_from_asyncio_queue(response.status)
         self.logger.info(f"Got response {status}")
-        return [{'id': 1}, {'id': 2}]
+        return [{"id": 1}, {"id": 2}]
 
 
 class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
@@ -133,113 +135,119 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
 
 class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
 
-    name = 'asyncdef_asyncio_reqs_return'
+    name = "asyncdef_asyncio_reqs_return"
 
     async def parse(self, response):
         await asyncio.sleep(0.2)
-        req_id = response.meta.get('req_id', 0)
+        req_id = response.meta.get("req_id", 0)
         status = await get_from_asyncio_queue(response.status)
         self.logger.info(f"Got response {status}, req_id {req_id}")
         if req_id > 0:
             return
         reqs = []
         for i in range(1, 3):
-            req = Request(self.start_urls[0], dont_filter=True, meta={'req_id': i})
+            req = Request(self.start_urls[0], dont_filter=True, meta={"req_id": i})
             reqs.append(req)
         return reqs
 
 
 class AsyncDefAsyncioGenExcSpider(SimpleSpider):
-    name = 'asyncdef_asyncio_gen_exc'
+    name = "asyncdef_asyncio_gen_exc"
 
     async def parse(self, response):
         for i in range(10):
             await asyncio.sleep(0.1)
-            yield {'foo': i}
+            yield {"foo": i}
             if i > 5:
                 raise ValueError("Stopping the processing")
 
 
 class AsyncDefDeferredDirectSpider(SimpleSpider):
-    name = 'asyncdef_deferred_direct'
+    name = "asyncdef_deferred_direct"
 
     async def parse(self, response):
         resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))
-        yield {'code': resp.code}
+        yield {"code": resp.code}
 
 
 class AsyncDefDeferredWrappedSpider(SimpleSpider):
-    name = 'asyncdef_deferred_wrapped'
+    name = "asyncdef_deferred_wrapped"
 
     async def parse(self, response):
-        resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
-        yield {'code': resp.code}
+        resp = await deferred_to_future(
+            get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        )
+        yield {"code": resp.code}
 
 
 class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
-    name = 'asyncdef_deferred_wrapped'
+    name = "asyncdef_deferred_wrapped"
 
     async def parse(self, response):
-        resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
-        yield {'code': resp.code}
+        resp = await maybe_deferred_to_future(
+            get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        )
+        yield {"code": resp.code}
 
 
 class AsyncDefAsyncioGenSpider(SimpleSpider):
 
-    name = 'asyncdef_asyncio_gen'
+    name = "asyncdef_asyncio_gen"
 
     async def parse(self, response):
         await asyncio.sleep(0.2)
-        yield {'foo': 42}
+        yield {"foo": 42}
         self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
 
-    name = 'asyncdef_asyncio_gen_loop'
+    name = "asyncdef_asyncio_gen_loop"
 
     async def parse(self, response):
         for i in range(10):
             await asyncio.sleep(0.1)
-            yield {'foo': i}
+            yield {"foo": i}
         self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
 
-    name = 'asyncdef_asyncio_gen_complex'
+    name = "asyncdef_asyncio_gen_complex"
     initial_reqs = 4
     following_reqs = 3
     depth = 2
 
     def _get_req(self, index, cb=None):
-        return Request(self.mockserver.url(f"/status?n=200&request={index}"),
-                       meta={'index': index},
-                       dont_filter=True,
-                       callback=cb)
+        return Request(
+            self.mockserver.url(f"/status?n=200&request={index}"),
+            meta={"index": index},
+            dont_filter=True,
+            callback=cb,
+        )
 
     def start_requests(self):
         for i in range(1, self.initial_reqs + 1):
             yield self._get_req(i)
 
     async def parse(self, response):
-        index = response.meta['index']
-        yield {'index': index}
-        if index < 10 ** self.depth:
+        index = response.meta["index"]
+        yield {"index": index}
+        if index < 10**self.depth:
             for new_index in range(10 * index, 10 * index + self.following_reqs):
                 yield self._get_req(new_index)
         yield self._get_req(index, cb=self.parse2)
         await asyncio.sleep(0.1)
-        yield {'index': index + 5}
+        yield {"index": index + 5}
 
     async def parse2(self, response):
         await asyncio.sleep(0.1)
-        yield {'index2': response.meta['index']}
+        yield {"index2": response.meta["index"]}
 
 
 class ItemSpider(FollowAllSpider):
 
-    name = 'item'
+    name = "item"
 
     def parse(self, response):
         for request in super().parse(response):
@@ -254,11 +262,11 @@ class DefaultError(Exception):
 
 class ErrorSpider(FollowAllSpider):
 
-    name = 'error'
+    name = "error"
     exception_cls = DefaultError
 
     def raise_exception(self):
-        raise self.exception_cls('Expected exception')
+        raise self.exception_cls("Expected exception")
 
     def parse(self, response):
         for request in super().parse(response):
@@ -280,16 +288,18 @@ class BrokenStartRequestsSpider(FollowAllSpider):
             1 / 0
 
         for s in range(100):
-            qargs = {'total': 10, 'seed': s}
+            qargs = {"total": 10, "seed": s}
             url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
-            yield Request(url, meta={'seed': s})
+            yield Request(url, meta={"seed": s})
             if self.fail_yielding:
                 2 / 0
 
-        assert self.seedsseen, 'All start requests consumed before any download happened'
+        assert (
+            self.seedsseen
+        ), "All start requests consumed before any download happened"
 
     def parse(self, response):
-        self.seedsseen.append(response.meta.get('seed'))
+        self.seedsseen.append(response.meta.get("seed"))
         for req in super().parse(response):
             yield req
 
@@ -307,21 +317,21 @@ class SingleRequestSpider(MetaSpider):
             yield Request(self.seed, callback=self.parse, errback=self.on_error)
 
     def parse(self, response):
-        self.meta.setdefault('responses', []).append(response)
+        self.meta.setdefault("responses", []).append(response)
         if callable(self.callback_func):
             return self.callback_func(response)
-        if 'next' in response.meta:
-            return response.meta['next']
+        if "next" in response.meta:
+            return response.meta["next"]
 
     def on_error(self, failure):
-        self.meta['failure'] = failure
+        self.meta["failure"] = failure
         if callable(self.errback_func):
             return self.errback_func(failure)
 
 
 class DuplicateStartRequestsSpider(MockServerSpider):
     dont_filter = True
-    name = 'duplicatestartrequests'
+    name = "duplicatestartrequests"
     distinct_urls = 2
     dupe_factor = 3
 
@@ -343,13 +353,12 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
     """
     A CrawlSpider which overrides the 'parse' method
     """
-    name = 'crawl_spider_with_parse_method'
+
+    name = "crawl_spider_with_parse_method"
     custom_settings: dict = {
-        'RETRY_HTTP_CODES': [],  # no need to retry
+        "RETRY_HTTP_CODES": [],  # no need to retry
     }
-    rules = (
-        Rule(LinkExtractor(), callback='parse', follow=True),
-    )
+    rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
 
     def start_requests(self):
         test_body = b"""
@@ -365,39 +374,45 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         yield Request(url, method="POST", body=test_body)
 
     def parse(self, response, foo=None):
-        self.logger.info('[parse] status %i (foo: %s)', response.status, foo)
-        yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
+        self.logger.info("[parse] status %i (foo: %s)", response.status, foo)
+        yield Request(
+            self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}
+        )
 
 
 class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
     """A CrawlSpider with an async def callback"""
-    name = 'crawl_spider_with_async_callback'
-    rules = (
-        Rule(LinkExtractor(), callback='parse_async', follow=True),
-    )
+
+    name = "crawl_spider_with_async_callback"
+    rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),)
 
     async def parse_async(self, response, foo=None):
-        self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo)
-        return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
+        self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo)
+        return Request(
+            self.mockserver.url("/status?n=202"),
+            self.parse_async,
+            cb_kwargs={"foo": "bar"},
+        )
 
 
 class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
     """A CrawlSpider with an async generator callback"""
-    name = 'crawl_spider_with_async_generator_callback'
-    rules = (
-        Rule(LinkExtractor(), callback='parse_async_gen', follow=True),
-    )
+
+    name = "crawl_spider_with_async_generator_callback"
+    rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),)
 
     async def parse_async_gen(self, response, foo=None):
-        self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo)
-        yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"})
+        self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo)
+        yield Request(
+            self.mockserver.url("/status?n=202"),
+            self.parse_async_gen,
+            cb_kwargs={"foo": "bar"},
+        )
 
 
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
-    name = 'crawl_spider_with_errback'
-    rules = (
-        Rule(LinkExtractor(), callback='parse', errback='errback', follow=True),
-    )
+    name = "crawl_spider_with_errback"
+    rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)
 
     def start_requests(self):
         test_body = b"""
@@ -416,13 +431,18 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
         yield Request(url, method="POST", body=test_body)
 
     def errback(self, failure):
-        self.logger.info('[errback] status %i', failure.value.response.status)
+        self.logger.info("[errback] status %i", failure.value.response.status)
 
 
 class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
-    name = 'crawl_spider_with_process_request_cb_kwargs'
+    name = "crawl_spider_with_process_request_cb_kwargs"
     rules = (
-        Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"),
+        Rule(
+            LinkExtractor(),
+            callback="parse",
+            follow=True,
+            process_request="process_request",
+        ),
     )
 
     def process_request(self, request, response):
@@ -457,14 +477,12 @@ class BytesReceivedCallbackSpider(MetaSpider):
 
 
 class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
-
     def bytes_received(self, data, request, spider):
         self.meta["bytes_received"] = data
         raise StopDownload(fail=True)
 
 
 class HeadersReceivedCallbackSpider(MetaSpider):
-
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
         spider = super().from_crawler(crawler, *args, **kwargs)
@@ -486,7 +504,6 @@ class HeadersReceivedCallbackSpider(MetaSpider):
 
 
 class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
-
     def headers_received(self, headers, body_length, request, spider):
         self.meta["headers_received"] = headers
         raise StopDownload(fail=True)
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index be8adadb3..c497450f7 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -6,7 +6,6 @@ from tests.mockserver import MockServer
 
 
 class TestCloseSpider(TestCase):
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -17,40 +16,40 @@ class TestCloseSpider(TestCase):
     @defer.inlineCallbacks
     def test_closespider_itemcount(self):
         close_on = 5
-        crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on})
+        crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
         yield crawler.crawl(mockserver=self.mockserver)
-        reason = crawler.spider.meta['close_reason']
-        self.assertEqual(reason, 'closespider_itemcount')
-        itemcount = crawler.stats.get_value('item_scraped_count')
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_itemcount")
+        itemcount = crawler.stats.get_value("item_scraped_count")
         self.assertTrue(itemcount >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_pagecount(self):
         close_on = 5
-        crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on})
+        crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
         yield crawler.crawl(mockserver=self.mockserver)
-        reason = crawler.spider.meta['close_reason']
-        self.assertEqual(reason, 'closespider_pagecount')
-        pagecount = crawler.stats.get_value('response_received_count')
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_pagecount")
+        pagecount = crawler.stats.get_value("response_received_count")
         self.assertTrue(pagecount >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):
         close_on = 5
-        crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on})
+        crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
-        reason = crawler.spider.meta['close_reason']
-        self.assertEqual(reason, 'closespider_errorcount')
-        key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}'
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_errorcount")
+        key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
         errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_timeout(self):
         close_on = 0.1
-        crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on})
+        crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
-        reason = crawler.spider.meta['close_reason']
-        self.assertEqual(reason, 'closespider_timeout')
-        total_seconds = crawler.stats.get_value('elapsed_time_seconds')
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_timeout")
+        total_seconds = crawler.stats.get_value("elapsed_time_seconds")
         self.assertTrue(total_seconds >= close_on)
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 87ab1ee81..5aa35a6d9 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -12,47 +12,52 @@ from scrapy.utils.test import get_testenv
 
 
 class CmdlineTest(unittest.TestCase):
-
     def setUp(self):
         self.env = get_testenv()
-        self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings'
+        self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
 
     def _execute(self, *new_args, **kwargs):
-        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
-        args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
+        encoding = getattr(sys.stdout, "encoding") or "utf-8"
+        args = (sys.executable, "-m", "scrapy.cmdline") + new_args
         proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
         comm = proc.communicate()[0].strip()
         return comm.decode(encoding)
 
     def test_default_settings(self):
-        self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default')
+        self.assertEqual(self._execute("settings", "--get", "TEST1"), "default")
 
     def test_override_settings_using_set_arg(self):
-        self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s',
-                                       'TEST1=override'), 'override')
+        self.assertEqual(
+            self._execute("settings", "--get", "TEST1", "-s", "TEST1=override"),
+            "override",
+        )
 
     def test_profiling(self):
         path = Path(tempfile.mkdtemp())
-        filename = path / 'res.prof'
+        filename = path / "res.prof"
         try:
-            self._execute('version', '--profile', str(filename))
+            self._execute("version", "--profile", str(filename))
             self.assertTrue(filename.exists())
             out = StringIO()
             stats = pstats.Stats(str(filename), stream=out)
             stats.print_stats()
             out.seek(0)
             stats = out.read()
-            self.assertIn(str(Path('scrapy', 'commands', 'version.py')),
-                          stats)
-            self.assertIn('tottime', stats)
+            self.assertIn(str(Path("scrapy", "commands", "version.py")), stats)
+            self.assertIn("tottime", stats)
         finally:
             shutil.rmtree(path)
 
     def test_override_dict_settings(self):
         EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
         EXTENSIONS = {EXT_PATH: 200}
-        settingsstr = self._execute('settings', '--get', 'EXTENSIONS', '-s',
-                                    'EXTENSIONS=' + json.dumps(EXTENSIONS))
+        settingsstr = self._execute(
+            "settings",
+            "--get",
+            "EXTENSIONS",
+            "-s",
+            "EXTENSIONS=" + json.dumps(EXTENSIONS),
+        )
         # XXX: There's gotta be a smarter way to do this...
         self.assertNotIn("...", settingsstr)
         for char in ("'", "<", ">"):
@@ -62,5 +67,7 @@ class CmdlineTest(unittest.TestCase):
         self.assertEqual(200, settingsdict[EXT_PATH])
 
     def test_pathlib_path_as_feeds_key(self):
-        self.assertEqual(self._execute('settings', '--get', 'FEEDS'),
-                         json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}))
+        self.assertEqual(
+            self._execute("settings", "--get", "FEEDS"),
+            json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}),
+        )
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index 005e45214..11c821f8d 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -2,9 +2,8 @@
 
 
 class TestExtension:
-
     def __init__(self, settings):
-        settings.set('TEST1', f"{settings['TEST1']} + started")
+        settings.set("TEST1", f"{settings['TEST1']} + started")
 
     @classmethod
     def from_crawler(cls, crawler):
diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py
index b0ac6e98b..32b15e191 100644
--- a/tests/test_cmdline/settings.py
+++ b/tests/test_cmdline/settings.py
@@ -1,14 +1,14 @@
 from pathlib import Path
 
 EXTENSIONS = {
-    'tests.test_cmdline.extensions.TestExtension': 0,
+    "tests.test_cmdline.extensions.TestExtension": 0,
 }
 
-TEST1 = 'default'
+TEST1 = "default"
 
 FEEDS = {
-    Path('items.csv'): {
-        'format': 'csv',
-        'fields': ['price', 'name'],
+    Path("items.csv"): {
+        "format": "csv",
+        "fields": ["price", "name"],
     },
 }
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
index fcafcef68..d5088e817 100644
--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -5,16 +5,15 @@ from subprocess import Popen, PIPE
 
 
 class CmdlineCrawlPipelineTest(unittest.TestCase):
-
     def _execute(self, spname):
-        args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)
+        args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
         cwd = Path(__file__).resolve().parent
         proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
         proc.communicate()
         return proc.returncode
 
     def test_open_spider_normally_in_pipeline(self):
-        self.assertEqual(self._execute('normal'), 0)
+        self.assertEqual(self._execute("normal"), 0)
 
     def test_exception_at_open_spider_in_pipeline(self):
-        self.assertEqual(self._execute('exception'), 1)
+        self.assertEqual(self._execute("exception"), 1)
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
index bd1f9cd8c..af15cac68 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
@@ -1,5 +1,4 @@
 class TestSpiderPipeline:
-
     def open_spider(self, spider):
         pass
 
@@ -8,9 +7,8 @@ class TestSpiderPipeline:
 
 
 class TestSpiderExceptionPipeline:
-
     def open_spider(self, spider):
-        raise Exception('exception')
+        raise Exception("exception")
 
     def process_item(self, item, spider):
         return item
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
index ae782c0d8..45c5f2eb9 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
@@ -1,2 +1,2 @@
-BOT_NAME = 'test_spider'
-SPIDER_MODULES = ['test_spider.spiders']
+BOT_NAME = "test_spider"
+SPIDER_MODULES = ["test_spider.spiders"]
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
index 300f45ebf..ebd4082c6 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
@@ -2,12 +2,10 @@ import scrapy
 
 
 class ExceptionSpider(scrapy.Spider):
-    name = 'exception'
+    name = "exception"
 
     custom_settings = {
-        'ITEM_PIPELINES': {
-            'test_spider.pipelines.TestSpiderExceptionPipeline': 300
-        }
+        "ITEM_PIPELINES": {"test_spider.pipelines.TestSpiderExceptionPipeline": 300}
     }
 
     def parse(self, response):
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
index 87a40fdcb..9a258c80d 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
@@ -2,12 +2,10 @@ import scrapy
 
 
 class NormalSpider(scrapy.Spider):
-    name = 'normal'
+    name = "normal"
 
     custom_settings = {
-        'ITEM_PIPELINES': {
-            'test_spider.pipelines.TestSpiderPipeline': 300
-        }
+        "ITEM_PIPELINES": {"test_spider.pipelines.TestSpiderPipeline": 300}
     }
 
     def parse(self, response):
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 049076e5c..1f299587f 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -3,15 +3,16 @@ from tests.test_commands import CommandTest
 
 class CheckCommandTest(CommandTest):
 
-    command = 'check'
+    command = "check"
 
     def setUp(self):
         super().setUp()
-        self.spider_name = 'check_spider'
-        self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()
+        self.spider_name = "check_spider"
+        self.spider = (self.proj_mod_path / "spiders" / "checkspider.py").resolve()
 
     def _write_contract(self, contracts, parse_def):
-        self.spider.write_text(f"""
+        self.spider.write_text(
+            f"""
 import scrapy
 
 class CheckSpider(scrapy.Spider):
@@ -24,13 +25,15 @@ class CheckSpider(scrapy.Spider):
         {contracts}
         \"\"\"
         {parse_def}
-        """, encoding="utf-8")
+        """,
+            encoding="utf-8",
+        )
 
-    def _test_contract(self, contracts='', parse_def='pass'):
+    def _test_contract(self, contracts="", parse_def="pass"):
         self._write_contract(contracts, parse_def)
-        p, out, err = self.proc('check')
-        self.assertNotIn('F', out)
-        self.assertIn('OK', err)
+        p, out, err = self.proc("check")
+        self.assertNotIn("F", out)
+        self.assertIn("OK", err)
         self.assertEqual(p.returncode, 0)
 
     def test_check_returns_requests_contract(self):
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 9d3c8fe73..bd44fa76e 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -7,28 +7,30 @@ from scrapy.utils.testproc import ProcessTest
 
 class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
 
-    command = 'fetch'
+    command = "fetch"
 
     @defer.inlineCallbacks
     def test_output(self):
-        _, out, _ = yield self.execute([self.url('/text')])
-        self.assertEqual(out.strip(), b'Works')
+        _, out, _ = yield self.execute([self.url("/text")])
+        self.assertEqual(out.strip(), b"Works")
 
     @defer.inlineCallbacks
     def test_redirect_default(self):
-        _, out, _ = yield self.execute([self.url('/redirect')])
-        self.assertEqual(out.strip(), b'Redirected here')
+        _, out, _ = yield self.execute([self.url("/redirect")])
+        self.assertEqual(out.strip(), b"Redirected here")
 
     @defer.inlineCallbacks
     def test_redirect_disabled(self):
-        _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')])
+        _, out, err = yield self.execute(
+            ["--no-redirect", self.url("/redirect-no-meta-refresh")]
+        )
         err = err.strip()
-        self.assertIn(b'downloader/response_status_count/302', err, err)
-        self.assertNotIn(b'downloader/response_status_count/200', err, err)
+        self.assertIn(b"downloader/response_status_count/302", err, err)
+        self.assertNotIn(b"downloader/response_status_count/200", err, err)
 
     @defer.inlineCallbacks
     def test_headers(self):
-        _, out, _ = yield self.execute([self.url('/text'), '--headers'])
-        out = out.replace(b'\r', b'')  # required on win32
-        assert b'Server: TwistedWeb' in out, out
-        assert b'Content-Type: text/plain' in out
+        _, out, _ = yield self.execute([self.url("/text"), "--headers"])
+        out = out.replace(b"\r", b"")  # required on win32
+        assert b"Server: TwistedWeb" in out, out
+        assert b"Content-Type: text/plain" in out
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 368b80513..1ee1bf5a7 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -14,16 +14,17 @@ from tests.test_commands import CommandTest
 def _textmode(bstr):
     """Normalize input the same as writing to a file
     and reading from it in text mode"""
-    return to_unicode(bstr).replace(os.linesep, '\n')
+    return to_unicode(bstr).replace(os.linesep, "\n")
 
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
-    command = 'parse'
+    command = "parse"
 
     def setUp(self):
         super().setUp()
-        self.spider_name = 'parse_spider'
-        (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f"""
+        self.spider_name = "parse_spider"
+        (self.proj_mod_path / "spiders" / "myspider.py").write_text(
+            f"""
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
@@ -92,9 +93,12 @@ class MyBadCrawlSpider(CrawlSpider):
 
     def parse(self, response):
         return [scrapy.Item(), dict(foo='bar')]
-""", encoding="utf-8")
+""",
+            encoding="utf-8",
+        )
 
-        (self.proj_mod_path / 'pipelines.py').write_text("""
+        (self.proj_mod_path / "pipelines.py").write_text(
+            """
 import logging
 
 class MyPipeline:
@@ -103,91 +107,141 @@ class MyPipeline:
     def process_item(self, item, spider):
         logging.info('It Works!')
         return item
-""", encoding="utf-8")
+""",
+            encoding="utf-8",
+        )
 
-        with (self.proj_mod_path / 'settings.py').open("a", encoding="utf-8") as f:
-            f.write(f"""
+        with (self.proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
+            f.write(
+                f"""
 ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
-""")
+"""
+            )
 
     @defer.inlineCallbacks
     def test_spider_arguments(self):
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '-a', 'test_arg=1',
-                                           '-c', 'parse',
-                                           '--verbose',
-                                           self.url('/html')])
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "-a",
+                "test_arg=1",
+                "-c",
+                "parse",
+                "--verbose",
+                self.url("/html"),
+            ]
+        )
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_with_meta(self):
         raw_json_string = '{"foo" : "baz"}'
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '--meta', raw_json_string,
-                                           '-c', 'parse_request_with_meta',
-                                           '--verbose',
-                                           self.url('/html')])
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "--meta",
+                raw_json_string,
+                "-c",
+                "parse_request_with_meta",
+                "--verbose",
+                self.url("/html"),
+            ]
+        )
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '-m', raw_json_string,
-                                           '-c', 'parse_request_with_meta',
-                                           '--verbose',
-                                           self.url('/html')])
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "-m",
+                raw_json_string,
+                "-c",
+                "parse_request_with_meta",
+                "--verbose",
+                self.url("/html"),
+            ]
+        )
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_with_cb_kwargs(self):
         raw_json_string = '{"foo" : "bar", "key": "value"}'
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '--cbkwargs', raw_json_string,
-                                           '-c', 'parse_request_with_cb_kwargs',
-                                           '--verbose',
-                                           self.url('/html')])
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "--cbkwargs",
+                raw_json_string,
+                "-c",
+                "parse_request_with_cb_kwargs",
+                "--verbose",
+                self.url("/html"),
+            ]
+        )
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '-c', 'parse_request_without_meta',
-                                           '--nolinks',
-                                           self.url('/html')])
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "-c",
+                "parse_request_without_meta",
+                "--nolinks",
+                self.url("/html"),
+            ]
+        )
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_pipelines(self):
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '--pipelines',
-                                           '-c', 'parse',
-                                           '--verbose',
-                                           self.url('/html')])
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "--pipelines",
+                "-c",
+                "parse",
+                "--verbose",
+                self.url("/html"),
+            ]
+        )
         self.assertIn("INFO: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_asyncio_parse_items(self):
         status, out, stderr = yield self.execute(
-            ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')]
+            [
+                "--spider",
+                "asyncdef" + self.spider_name,
+                "-c",
+                "parse",
+                self.url("/html"),
+            ]
         )
         self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_parse_items(self):
         status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
+            ["--spider", self.spider_name, "-c", "parse", self.url("/html")]
         )
         self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_parse_items_no_callback_passed(self):
         status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, self.url('/html')]
+            ["--spider", self.spider_name, self.url("/html")]
         )
         self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_wrong_callback_passed(self):
         status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
+            ["--spider", self.spider_name, "-c", "dummy", self.url("/html")]
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find callback""", _textmode(stderr))
@@ -196,7 +250,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
     def test_crawlspider_matching_rule_callback_set(self):
         """If a rule matches the URL, use it's defined callback."""
         status, out, stderr = yield self.execute(
-            ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/html')]
+            ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")]
         )
         self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
@@ -204,7 +258,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
     def test_crawlspider_matching_rule_default_callback(self):
         """If a rule match but it has no callback set, use the 'parse' callback."""
         status, out, stderr = yield self.execute(
-            ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/text')]
+            ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")]
         )
         self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
 
@@ -212,7 +266,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
     def test_spider_with_no_rules_attribute(self):
         """Using -r with a spider with no rule should not produce items."""
         status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, '-r', self.url('/html')]
+            ["--spider", self.spider_name, "-r", self.url("/html")]
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
@@ -220,7 +274,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
     @defer.inlineCallbacks
     def test_crawlspider_missing_callback(self):
         status, out, stderr = yield self.execute(
-            ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/html')]
+            ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")]
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
 
@@ -228,14 +282,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
     def test_crawlspider_no_matching_rule(self):
         """The requested URL has no matching rule, so no items should be scraped"""
         status, out, stderr = yield self.execute(
-            ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/enc-gb18030')]
+            ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")]
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_crawlspider_not_exists_with_not_matched_url(self):
-        status, out, stderr = yield self.execute([self.url('/invalid_url')])
+        status, out, stderr = yield self.execute([self.url("/invalid_url")])
         self.assertEqual(status, 0)
 
     @defer.inlineCallbacks
@@ -243,14 +297,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         """Checks if a file was created successfully having
         correct format containing correct data in it.
         """
-        file_name = 'data.json'
+        file_name = "data.json"
         file_path = Path(self.proj_path, file_name)
-        yield self.execute([
-            '--spider', self.spider_name,
-            '-c', 'parse',
-            '-o', file_name,
-            self.url('/html')
-        ])
+        yield self.execute(
+            [
+                "--spider",
+                self.spider_name,
+                "-c",
+                "parse",
+                "-o",
+                file_name,
+                self.url("/html"),
+            ]
+        )
 
         self.assertTrue(file_path.exists())
         self.assertTrue(file_path.is_file())
@@ -262,12 +321,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         command = parse.Command()
         command.settings = Settings()
         parser = argparse.ArgumentParser(
-            prog='scrapy', formatter_class=argparse.HelpFormatter,
-            conflict_handler='resolve', prefix_chars='-'
+            prog="scrapy",
+            formatter_class=argparse.HelpFormatter,
+            conflict_handler="resolve",
+            prefix_chars="-",
         )
         command.add_options(parser)
         namespace = parser.parse_args(
-            ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
+            ["--verbose", "--nolinks", "-d", "2", "--spider", self.spider_name]
         )
         self.assertTrue(namespace.nolinks)
         self.assertEqual(namespace.depth, 2)
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 33c98ad69..4c4242a1b 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -11,107 +11,118 @@ from tests import tests_datadir, NON_EXISTING_RESOLVABLE
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
-    command = 'shell'
+    command = "shell"
 
     @defer.inlineCallbacks
     def test_empty(self):
-        _, out, _ = yield self.execute(['-c', 'item'])
-        assert b'{}' in out
+        _, out, _ = yield self.execute(["-c", "item"])
+        assert b"{}" in out
 
     @defer.inlineCallbacks
     def test_response_body(self):
-        _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
-        assert b'Works' in out
+        _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"])
+        assert b"Works" in out
 
     @defer.inlineCallbacks
     def test_response_type_text(self):
-        _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
-        assert b'TextResponse' in out
+        _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"])
+        assert b"TextResponse" in out
 
     @defer.inlineCallbacks
     def test_response_type_html(self):
-        _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
-        assert b'HtmlResponse' in out
+        _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"])
+        assert b"HtmlResponse" in out
 
     @defer.inlineCallbacks
     def test_response_selector_html(self):
-        xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
-        _, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
-        self.assertEqual(out.strip(), b'Works')
+        xpath = "response.xpath(\"//p[@class='one']/text()\").get()"
+        _, out, _ = yield self.execute([self.url("/html"), "-c", xpath])
+        self.assertEqual(out.strip(), b"Works")
 
     @defer.inlineCallbacks
     def test_response_encoding_gb18030(self):
-        _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
-        self.assertEqual(out.strip(), b'gb18030')
+        _, out, _ = yield self.execute(
+            [self.url("/enc-gb18030"), "-c", "response.encoding"]
+        )
+        self.assertEqual(out.strip(), b"gb18030")
 
     @defer.inlineCallbacks
     def test_redirect(self):
-        _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
-        assert out.strip().endswith(b'/redirected')
+        _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"])
+        assert out.strip().endswith(b"/redirected")
 
     @defer.inlineCallbacks
     def test_redirect_follow_302(self):
-        _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
-        assert out.strip().endswith(b'200')
+        _, out, _ = yield self.execute(
+            [self.url("/redirect-no-meta-refresh"), "-c", "response.status"]
+        )
+        assert out.strip().endswith(b"200")
 
     @defer.inlineCallbacks
     def test_redirect_not_follow_302(self):
         _, out, _ = yield self.execute(
-            ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']
+            [
+                "--no-redirect",
+                self.url("/redirect-no-meta-refresh"),
+                "-c",
+                "response.status",
+            ]
         )
-        assert out.strip().endswith(b'302')
+        assert out.strip().endswith(b"302")
 
     @defer.inlineCallbacks
     def test_fetch_redirect_follow_302(self):
         """Test that calling ``fetch(url)`` follows HTTP redirects by default."""
-        url = self.url('/redirect-no-meta-refresh')
+        url = self.url("/redirect-no-meta-refresh")
         code = f"fetch('{url}')"
-        errcode, out, errout = yield self.execute(['-c', code])
+        errcode, out, errout = yield self.execute(["-c", code])
         self.assertEqual(errcode, 0, out)
-        assert b'Redirecting (302)' in errout
-        assert b'Crawled (200)' in errout
+        assert b"Redirecting (302)" in errout
+        assert b"Crawled (200)" in errout
 
     @defer.inlineCallbacks
     def test_fetch_redirect_not_follow_302(self):
         """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects."""
-        url = self.url('/redirect-no-meta-refresh')
+        url = self.url("/redirect-no-meta-refresh")
         code = f"fetch('{url}', redirect=False)"
-        errcode, out, errout = yield self.execute(['-c', code])
+        errcode, out, errout = yield self.execute(["-c", code])
         self.assertEqual(errcode, 0, out)
-        assert b'Crawled (302)' in errout
+        assert b"Crawled (302)" in errout
 
     @defer.inlineCallbacks
     def test_request_replace(self):
-        url = self.url('/text')
+        url = self.url("/text")
         code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))"
-        errcode, out, _ = yield self.execute(['-c', code])
+        errcode, out, _ = yield self.execute(["-c", code])
         self.assertEqual(errcode, 0, out)
 
     @defer.inlineCallbacks
     def test_scrapy_import(self):
-        url = self.url('/text')
+        url = self.url("/text")
         code = f"fetch(scrapy.Request('{url}'))"
-        errcode, out, _ = yield self.execute(['-c', code])
+        errcode, out, _ = yield self.execute(["-c", code])
         self.assertEqual(errcode, 0, out)
 
     @defer.inlineCallbacks
     def test_local_file(self):
-        filepath = Path(tests_datadir, 'test_site', 'index.html')
-        _, out, _ = yield self.execute([str(filepath), '-c', 'item'])
-        assert b'{}' in out
+        filepath = Path(tests_datadir, "test_site", "index.html")
+        _, out, _ = yield self.execute([str(filepath), "-c", "item"])
+        assert b"{}" in out
 
     @defer.inlineCallbacks
     def test_local_nofile(self):
-        filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
-        errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False)
+        filepath = "file:///tests/sample_data/test_site/nothinghere.html"
+        errcode, out, err = yield self.execute(
+            [filepath, "-c", "item"], check_code=False
+        )
         self.assertEqual(errcode, 1, out or err)
-        self.assertIn(b'No such file or directory', err)
+        self.assertIn(b"No such file or directory", err)
 
     @defer.inlineCallbacks
     def test_dns_failures(self):
         if NON_EXISTING_RESOLVABLE:
             raise unittest.SkipTest("Non-existing hosts are resolvable")
-        url = 'www.somedomainthatdoesntexi.st'
-        errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
+        url = "www.somedomainthatdoesntexi.st"
+        errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False)
         self.assertEqual(errcode, 1, out or err)
-        self.assertIn(b'DNS lookup failed', err)
+        self.assertIn(b"DNS lookup failed", err)
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 00d998388..f97a088a8 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -8,11 +8,11 @@ from scrapy.utils.testproc import ProcessTest
 
 class VersionTest(ProcessTest, unittest.TestCase):
 
-    command = 'version'
+    command = "version"
 
     @defer.inlineCallbacks
     def test_output(self):
-        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
+        encoding = getattr(sys.stdout, "encoding") or "utf-8"
         _, out, _ = yield self.execute([])
         self.assertEqual(
             out.strip().decode(encoding),
@@ -21,13 +21,25 @@ class VersionTest(ProcessTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_verbose_output(self):
-        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
-        _, out, _ = yield self.execute(['-v'])
+        encoding = getattr(sys.stdout, "encoding") or "utf-8"
+        _, out, _ = yield self.execute(["-v"])
         headers = [
             line.partition(":")[0].strip()
             for line in out.strip().decode(encoding).splitlines()
         ]
-        self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
-                                   'cssselect', 'parsel', 'w3lib',
-                                   'Twisted', 'Python', 'pyOpenSSL',
-                                   'cryptography', 'Platform'])
+        self.assertEqual(
+            headers,
+            [
+                "Scrapy",
+                "lxml",
+                "libxml2",
+                "cssselect",
+                "parsel",
+                "w3lib",
+                "Twisted",
+                "Python",
+                "pyOpenSSL",
+                "cryptography",
+                "Platform",
+            ],
+        )
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 91476abf8..ed0eb85f5 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -33,35 +33,46 @@ from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
 class CommandSettings(unittest.TestCase):
-
     def setUp(self):
         self.command = ScrapyCommand()
         self.command.settings = Settings()
-        self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
-                                              conflict_handler='resolve')
+        self.parser = argparse.ArgumentParser(
+            formatter_class=ScrapyHelpFormatter, conflict_handler="resolve"
+        )
         self.command.add_options(self.parser)
 
     def test_settings_json_string(self):
         feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
-        opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
+        opts, args = self.parser.parse_known_args(
+            args=["-s", f"FEEDS={feeds_json}", "spider.py"]
+        )
         self.command.process_options(args, opts)
-        self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
-        self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
+        self.assertIsInstance(
+            self.command.settings["FEEDS"], scrapy.settings.BaseSettings
+        )
+        self.assertEqual(dict(self.command.settings["FEEDS"]), json.loads(feeds_json))
 
     def test_help_formatter(self):
-        formatter = ScrapyHelpFormatter(prog='scrapy')
-        part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
-                        '\n', 'optional arguments:\n', '\n', 'Global Options:\n']
+        formatter = ScrapyHelpFormatter(prog="scrapy")
+        part_strings = [
+            "usage: scrapy genspider [options] <name> <domain>\n\n",
+            "\n",
+            "optional arguments:\n",
+            "\n",
+            "Global Options:\n",
+        ]
         self.assertEqual(
             formatter._join_parts(part_strings),
-            ('Usage\n=====\n  scrapy genspider [options] <name> <domain>\n\n\n'
-             'Optional Arguments\n==================\n\n'
-             'Global Options\n--------------\n')
+            (
+                "Usage\n=====\n  scrapy genspider [options] <name> <domain>\n\n\n"
+                "Optional Arguments\n==================\n\n"
+                "Global Options\n--------------\n"
+            ),
         )
 
 
 class ProjectTest(unittest.TestCase):
-    project_name = 'testproject'
+    project_name = "testproject"
 
     def setUp(self):
         self.temp_path = mkdtemp()
@@ -75,15 +86,16 @@ class ProjectTest(unittest.TestCase):
 
     def call(self, *new_args, **kwargs):
         with tempfile.TemporaryFile() as out:
-            args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
-            return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd,
-                                   env=self.env, **kwargs)
+            args = (sys.executable, "-m", "scrapy.cmdline") + new_args
+            return subprocess.call(
+                args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs
+            )
 
     def proc(self, *new_args, **popen_kwargs):
-        args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
+        args = (sys.executable, "-m", "scrapy.cmdline") + new_args
         p = subprocess.Popen(
             args,
-            cwd=popen_kwargs.pop('cwd', self.cwd),
+            cwd=popen_kwargs.pop("cwd", self.cwd),
             env=self.env,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
@@ -93,7 +105,7 @@ class ProjectTest(unittest.TestCase):
         def kill_proc():
             p.kill()
             p.communicate()
-            assert False, 'Command took too much time to complete'
+            assert False, "Command took too much time to complete"
 
         timer = Timer(15, kill_proc)
         try:
@@ -104,7 +116,9 @@ class ProjectTest(unittest.TestCase):
 
         return p, to_unicode(stdout), to_unicode(stderr)
 
-    def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]:
+    def find_in_file(
+        self, filename: Union[str, os.PathLike], regex
+    ) -> Optional[re.Match]:
         """Find first pattern occurrence in file"""
         pattern = re.compile(regex)
         with Path(filename).open("r", encoding="utf-8") as f:
@@ -116,68 +130,75 @@ class ProjectTest(unittest.TestCase):
 
 
 class StartprojectTest(ProjectTest):
-
     def test_startproject(self):
-        p, out, err = self.proc('startproject', self.project_name)
+        p, out, err = self.proc("startproject", self.project_name)
         print(out)
         print(err, file=sys.stderr)
         self.assertEqual(p.returncode, 0)
 
-        assert Path(self.proj_path, 'scrapy.cfg').exists()
-        assert Path(self.proj_path, 'testproject').exists()
-        assert Path(self.proj_mod_path, '__init__.py').exists()
-        assert Path(self.proj_mod_path, 'items.py').exists()
-        assert Path(self.proj_mod_path, 'pipelines.py').exists()
-        assert Path(self.proj_mod_path, 'settings.py').exists()
-        assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists()
+        assert Path(self.proj_path, "scrapy.cfg").exists()
+        assert Path(self.proj_path, "testproject").exists()
+        assert Path(self.proj_mod_path, "__init__.py").exists()
+        assert Path(self.proj_mod_path, "items.py").exists()
+        assert Path(self.proj_mod_path, "pipelines.py").exists()
+        assert Path(self.proj_mod_path, "settings.py").exists()
+        assert Path(self.proj_mod_path, "spiders", "__init__.py").exists()
 
-        self.assertEqual(1, self.call('startproject', self.project_name))
-        self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
-        self.assertEqual(1, self.call('startproject', 'sys'))
+        self.assertEqual(1, self.call("startproject", self.project_name))
+        self.assertEqual(1, self.call("startproject", "wrong---project---name"))
+        self.assertEqual(1, self.call("startproject", "sys"))
 
     def test_startproject_with_project_dir(self):
         project_dir = mkdtemp()
-        self.assertEqual(0, self.call('startproject', self.project_name, project_dir))
+        self.assertEqual(0, self.call("startproject", self.project_name, project_dir))
 
-        assert Path(project_dir, 'scrapy.cfg').exists()
-        assert Path(project_dir, 'testproject').exists()
-        assert Path(project_dir, self.project_name, '__init__.py').exists()
-        assert Path(project_dir, self.project_name, 'items.py').exists()
-        assert Path(project_dir, self.project_name, 'pipelines.py').exists()
-        assert Path(project_dir, self.project_name, 'settings.py').exists()
-        assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists()
+        assert Path(project_dir, "scrapy.cfg").exists()
+        assert Path(project_dir, "testproject").exists()
+        assert Path(project_dir, self.project_name, "__init__.py").exists()
+        assert Path(project_dir, self.project_name, "items.py").exists()
+        assert Path(project_dir, self.project_name, "pipelines.py").exists()
+        assert Path(project_dir, self.project_name, "settings.py").exists()
+        assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists()
 
-        self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
+        self.assertEqual(
+            0, self.call("startproject", self.project_name, project_dir + "2")
+        )
 
-        self.assertEqual(1, self.call('startproject', self.project_name, project_dir))
-        self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir))
-        self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
-        self.assertEqual(1, self.call('startproject', 'sys'))
-        self.assertEqual(2, self.call('startproject'))
-        self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
+        self.assertEqual(1, self.call("startproject", self.project_name, project_dir))
+        self.assertEqual(
+            1, self.call("startproject", self.project_name + "2", project_dir)
+        )
+        self.assertEqual(1, self.call("startproject", "wrong---project---name"))
+        self.assertEqual(1, self.call("startproject", "sys"))
+        self.assertEqual(2, self.call("startproject"))
+        self.assertEqual(
+            2,
+            self.call("startproject", self.project_name, project_dir, "another_params"),
+        )
 
     def test_existing_project_dir(self):
         project_dir = mkdtemp()
-        project_name = self.project_name + '_existing'
+        project_name = self.project_name + "_existing"
         project_path = Path(project_dir, project_name)
         project_path.mkdir()
 
-        p, out, err = self.proc('startproject', project_name, cwd=project_dir)
+        p, out, err = self.proc("startproject", project_name, cwd=project_dir)
         print(out)
         print(err, file=sys.stderr)
         self.assertEqual(p.returncode, 0)
 
-        assert Path(project_path, 'scrapy.cfg').exists()
+        assert Path(project_path, "scrapy.cfg").exists()
         assert Path(project_path, project_name).exists()
-        assert Path(project_path, project_name, '__init__.py').exists()
-        assert Path(project_path, project_name, 'items.py').exists()
-        assert Path(project_path, project_name, 'pipelines.py').exists()
-        assert Path(project_path, project_name, 'settings.py').exists()
-        assert Path(project_path, project_name, 'spiders', '__init__.py').exists()
+        assert Path(project_path, project_name, "__init__.py").exists()
+        assert Path(project_path, project_name, "items.py").exists()
+        assert Path(project_path, project_name, "pipelines.py").exists()
+        assert Path(project_path, project_name, "settings.py").exists()
+        assert Path(project_path, project_name, "spiders", "__init__.py").exists()
 
 
-def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=None) -> Dict[str, str]:
-
+def get_permissions_dict(
+    path: Union[str, os.PathLike], renamings=None, ignore=None
+) -> Dict[str, str]:
     def get_permissions(path: Path) -> str:
         return oct(path.stat().st_mode)
 
@@ -185,7 +206,7 @@ def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=N
 
     renamings = renamings or tuple()
     permissions_dict = {
-        '.': get_permissions(path_obj),
+        ".": get_permissions(path_obj),
     }
     for root, dirs, files in os.walk(path_obj):
         nodes = list(chain(dirs, files))
@@ -196,10 +217,7 @@ def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=N
             absolute_path = Path(root, node)
             relative_path = str(absolute_path.relative_to(path))
             for search_string, replacement in renamings:
-                relative_path = relative_path.replace(
-                    search_string,
-                    replacement
-                )
+                relative_path = relative_path.replace(search_string, replacement)
             permissions = get_permissions(absolute_path)
             permissions_dict[relative_path] = permissions
     return permissions_dict
@@ -211,31 +229,33 @@ class StartprojectTemplatesTest(ProjectTest):
 
     def setUp(self):
         super().setUp()
-        self.tmpl = str(Path(self.temp_path, 'templates'))
-        self.tmpl_proj = str(Path(self.tmpl, 'project'))
+        self.tmpl = str(Path(self.temp_path, "templates"))
+        self.tmpl_proj = str(Path(self.tmpl, "project"))
 
     def test_startproject_template_override(self):
-        copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl)
-        Path(self.tmpl_proj, 'root_template').write_bytes(b"")
-        assert Path(self.tmpl_proj, 'root_template').exists()
+        copytree(Path(scrapy.__path__[0], "templates"), self.tmpl)
+        Path(self.tmpl_proj, "root_template").write_bytes(b"")
+        assert Path(self.tmpl_proj, "root_template").exists()
 
-        args = ['--set', f'TEMPLATES_DIR={self.tmpl}']
-        p, out, err = self.proc('startproject', self.project_name, *args)
-        self.assertIn(f"New Scrapy project '{self.project_name}', "
-                      "using template directory", out)
+        args = ["--set", f"TEMPLATES_DIR={self.tmpl}"]
+        p, out, err = self.proc("startproject", self.project_name, *args)
+        self.assertIn(
+            f"New Scrapy project '{self.project_name}', " "using template directory",
+            out,
+        )
         self.assertIn(self.tmpl_proj, out)
-        assert Path(self.proj_path, 'root_template').exists()
+        assert Path(self.proj_path, "root_template").exists()
 
     def test_startproject_permissions_from_writable(self):
         """Check that generated files have the right permissions when the
         template folder has the same permissions as in the project, i.e.
         everything is writable."""
         scrapy_path = scrapy.__path__[0]
-        project_template = Path(scrapy_path, 'templates', 'project')
-        project_name = 'startproject1'
+        project_template = Path(scrapy_path, "templates", "project")
+        project_name = "startproject1"
         renamings = (
-            ('module', project_name),
-            ('.tmpl', ''),
+            ("module", project_name),
+            (".tmpl", ""),
         )
         expected_permissions = get_permissions_dict(
             project_template,
@@ -247,9 +267,9 @@ class StartprojectTemplatesTest(ProjectTest):
         process = subprocess.Popen(
             (
                 sys.executable,
-                '-m',
-                'scrapy.cmdline',
-                'startproject',
+                "-m",
+                "scrapy.cmdline",
+                "startproject",
                 project_name,
             ),
             cwd=destination,
@@ -270,12 +290,12 @@ class StartprojectTemplatesTest(ProjectTest):
         See https://github.com/scrapy/scrapy/pull/4604
         """
         scrapy_path = scrapy.__path__[0]
-        templates_dir = Path(scrapy_path, 'templates')
-        project_template = Path(templates_dir, 'project')
-        project_name = 'startproject2'
+        templates_dir = Path(scrapy_path, "templates")
+        project_template = Path(templates_dir, "project")
+        project_name = "startproject2"
         renamings = (
-            ('module', project_name),
-            ('.tmpl', ''),
+            ("module", project_name),
+            (".tmpl", ""),
         )
         expected_permissions = get_permissions_dict(
             project_template,
@@ -287,7 +307,7 @@ class StartprojectTemplatesTest(ProjectTest):
             current_permissions = path.stat().st_mode
             path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION)
 
-        read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
+        read_only_templates_dir = str(Path(mkdtemp()) / "templates")
         copytree(templates_dir, read_only_templates_dir)
 
         for root, dirs, files in os.walk(read_only_templates_dir):
@@ -298,12 +318,12 @@ class StartprojectTemplatesTest(ProjectTest):
         process = subprocess.Popen(
             (
                 sys.executable,
-                '-m',
-                'scrapy.cmdline',
-                'startproject',
+                "-m",
+                "scrapy.cmdline",
+                "startproject",
                 project_name,
-                '--set',
-                f'TEMPLATES_DIR={read_only_templates_dir}',
+                "--set",
+                f"TEMPLATES_DIR={read_only_templates_dir}",
             ),
             cwd=destination,
             env=self.env,
@@ -319,11 +339,11 @@ class StartprojectTemplatesTest(ProjectTest):
         """Check that pre-existing folders and files in the destination folder
         do not see their permissions modified."""
         scrapy_path = scrapy.__path__[0]
-        project_template = Path(scrapy_path, 'templates', 'project')
-        project_name = 'startproject3'
+        project_template = Path(scrapy_path, "templates", "project")
+        project_name = "startproject3"
         renamings = (
-            ('module', project_name),
-            ('.tmpl', ''),
+            ("module", project_name),
+            (".tmpl", ""),
         )
         expected_permissions = get_permissions_dict(
             project_template,
@@ -336,15 +356,20 @@ class StartprojectTemplatesTest(ProjectTest):
 
         existing_nodes = {
             oct(permissions)[2:] + extension: permissions
-            for extension in ('', '.d')
+            for extension in ("", ".d")
             for permissions in (
-                0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
+                0o444,
+                0o555,
+                0o644,
+                0o666,
+                0o755,
+                0o777,
             )
         }
         project_dir.mkdir()
         for node, permissions in existing_nodes.items():
             path = project_dir / node
-            if node.endswith('.d'):
+            if node.endswith(".d"):
                 path.mkdir(mode=permissions)
             else:
                 path.touch(mode=permissions)
@@ -353,11 +378,11 @@ class StartprojectTemplatesTest(ProjectTest):
         process = subprocess.Popen(
             (
                 sys.executable,
-                '-m',
-                'scrapy.cmdline',
-                'startproject',
+                "-m",
+                "scrapy.cmdline",
+                "startproject",
                 project_name,
-                '.',
+                ".",
             ),
             cwd=project_dir,
             env=self.env,
@@ -372,6 +397,7 @@ class StartprojectTemplatesTest(ProjectTest):
         """Check that generated files have the right permissions when the
         system uses a umask value that causes new files to have different
         permissions than those from the template folder."""
+
         @contextmanager
         def umask(new_mask):
             cur_mask = os.umask(new_mask)
@@ -379,15 +405,11 @@ class StartprojectTemplatesTest(ProjectTest):
             os.umask(cur_mask)
 
         scrapy_path = scrapy.__path__[0]
-        project_template = Path(
-            scrapy_path,
-            'templates',
-            'project'
-        )
-        project_name = 'umaskproject'
+        project_template = Path(scrapy_path, "templates", "project")
+        project_name = "umaskproject"
         renamings = (
-            ('module', project_name),
-            ('.tmpl', ''),
+            ("module", project_name),
+            (".tmpl", ""),
         )
         expected_permissions = get_permissions_dict(
             project_template,
@@ -400,9 +422,9 @@ class StartprojectTemplatesTest(ProjectTest):
             process = subprocess.Popen(
                 (
                     sys.executable,
-                    '-m',
-                    'scrapy.cmdline',
-                    'startproject',
+                    "-m",
+                    "scrapy.cmdline",
+                    "startproject",
                     project_name,
                 ),
                 cwd=destination,
@@ -417,67 +439,74 @@ class StartprojectTemplatesTest(ProjectTest):
 
 
 class CommandTest(ProjectTest):
-
     def setUp(self):
         super().setUp()
-        self.call('startproject', self.project_name)
+        self.call("startproject", self.project_name)
         self.cwd = Path(self.temp_path, self.project_name)
-        self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings'
+        self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings"
 
 
 class GenspiderCommandTest(CommandTest):
-
     def test_arguments(self):
         # only pass one argument. spider script shouldn't be created
-        self.assertEqual(2, self.call('genspider', 'test_name'))
-        assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
+        self.assertEqual(2, self.call("genspider", "test_name"))
+        assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists()
         # pass two arguments <name> <domain>. spider script should be created
-        self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
-        assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
+        self.assertEqual(0, self.call("genspider", "test_name", "test.com"))
+        assert Path(self.proj_mod_path, "spiders", "test_name.py").exists()
 
-    def test_template(self, tplname='crawl'):
-        args = [f'--template={tplname}'] if tplname else []
-        spname = 'test_spider'
+    def test_template(self, tplname="crawl"):
+        args = [f"--template={tplname}"] if tplname else []
+        spname = "test_spider"
         spmodule = f"{self.project_name}.spiders.{spname}"
-        p, out, err = self.proc('genspider', spname, 'test.com', *args)
-        self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep}  {spmodule}", out)
-        self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists())
-        modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
-        p, out, err = self.proc('genspider', spname, 'test.com', *args)
+        p, out, err = self.proc("genspider", spname, "test.com", *args)
+        self.assertIn(
+            f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep}  {spmodule}",
+            out,
+        )
+        self.assertTrue(Path(self.proj_mod_path, "spiders", "test_spider.py").exists())
+        modify_time_before = (
+            Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime
+        )
+        p, out, err = self.proc("genspider", spname, "test.com", *args)
         self.assertIn(f"Spider {spname!r} already exists in module", out)
-        modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
+        modify_time_after = (
+            Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime
+        )
         self.assertEqual(modify_time_after, modify_time_before)
 
     def test_template_basic(self):
-        self.test_template('basic')
+        self.test_template("basic")
 
     def test_template_csvfeed(self):
-        self.test_template('csvfeed')
+        self.test_template("csvfeed")
 
     def test_template_xmlfeed(self):
-        self.test_template('xmlfeed')
+        self.test_template("xmlfeed")
 
     def test_list(self):
-        self.assertEqual(0, self.call('genspider', '--list'))
+        self.assertEqual(0, self.call("genspider", "--list"))
 
     def test_dump(self):
-        self.assertEqual(0, self.call('genspider', '--dump=basic'))
-        self.assertEqual(0, self.call('genspider', '-d', 'basic'))
+        self.assertEqual(0, self.call("genspider", "--dump=basic"))
+        self.assertEqual(0, self.call("genspider", "-d", "basic"))
 
     def test_same_name_as_project(self):
-        self.assertEqual(2, self.call('genspider', self.project_name))
-        assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists()
+        self.assertEqual(2, self.call("genspider", self.project_name))
+        assert not Path(
+            self.proj_mod_path, "spiders", f"{self.project_name}.py"
+        ).exists()
 
     def test_same_filename_as_existing_spider(self, force=False):
-        file_name = 'example'
-        file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py')
-        self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
+        file_name = "example"
+        file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py")
+        self.assertEqual(0, self.call("genspider", file_name, "example.com"))
         assert file_path.exists()
 
         # change name of spider but not its file name
-        with file_path.open('r+', encoding="utf-8") as spider_file:
+        with file_path.open("r+", encoding="utf-8") as spider_file:
             file_data = spider_file.read()
-            file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
+            file_data = file_data.replace("name = 'example'", "name = 'renamed'")
             spider_file.seek(0)
             spider_file.write(file_data)
             spider_file.truncate()
@@ -485,14 +514,16 @@ class GenspiderCommandTest(CommandTest):
         file_contents_before = file_data
 
         if force:
-            p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
-            self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out)
+            p, out, err = self.proc("genspider", "--force", file_name, "example.com")
+            self.assertIn(
+                f"Created spider {file_name!r} using template 'basic' in module", out
+            )
             modify_time_after = file_path.stat().st_mtime
             self.assertNotEqual(modify_time_after, modify_time_before)
             file_contents_after = file_path.read_text(encoding="utf-8")
             self.assertNotEqual(file_contents_after, file_contents_before)
         else:
-            p, out, err = self.proc('genspider', file_name, 'example.com')
+            p, out, err = self.proc("genspider", file_name, "example.com")
             self.assertIn(f"{file_path.resolve()} already exists", out)
             modify_time_after = file_path.stat().st_mtime
             self.assertEqual(modify_time_after, modify_time_before)
@@ -502,53 +533,63 @@ class GenspiderCommandTest(CommandTest):
     def test_same_filename_as_existing_spider_force(self):
         self.test_same_filename_as_existing_spider(force=True)
 
-    def test_url(self, url='test.com', domain="test.com"):
-        self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))
-        self.assertEqual(domain,
-                         self.find_in_file(Path(self.proj_mod_path,
-                                                'spiders', 'test_name.py'),
-                                           r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
-        self.assertEqual(f'http://{domain}/',
-                         self.find_in_file(Path(self.proj_mod_path,
-                                                'spiders', 'test_name.py'),
-                                           r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
+    def test_url(self, url="test.com", domain="test.com"):
+        self.assertEqual(0, self.call("genspider", "--force", "test_name", url))
+        self.assertEqual(
+            domain,
+            self.find_in_file(
+                Path(self.proj_mod_path, "spiders", "test_name.py"),
+                r"allowed_domains\s*=\s*\[\'(.+)\'\]",
+            ).group(1),
+        )
+        self.assertEqual(
+            f"http://{domain}/",
+            self.find_in_file(
+                Path(self.proj_mod_path, "spiders", "test_name.py"),
+                r"start_urls\s*=\s*\[\'(.+)\'\]",
+            ).group(1),
+        )
 
     def test_url_schema(self):
-        self.test_url('http://test.com', 'test.com')
+        self.test_url("http://test.com", "test.com")
 
     def test_url_path(self):
-        self.test_url('test.com/some/other/page', 'test.com')
+        self.test_url("test.com/some/other/page", "test.com")
 
     def test_url_schema_path(self):
-        self.test_url('https://test.com/some/other/page', 'test.com')
+        self.test_url("https://test.com/some/other/page", "test.com")
 
 
 class GenspiderStandaloneCommandTest(ProjectTest):
-
     def test_generate_standalone_spider(self):
-        self.call('genspider', 'example', 'example.com')
-        assert Path(self.temp_path, 'example.py').exists()
+        self.call("genspider", "example", "example.com")
+        assert Path(self.temp_path, "example.py").exists()
 
     def test_same_name_as_existing_file(self, force=False):
-        file_name = 'example'
-        file_path = Path(self.temp_path, file_name + '.py')
-        p, out, err = self.proc('genspider', file_name, 'example.com')
-        self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out)
+        file_name = "example"
+        file_path = Path(self.temp_path, file_name + ".py")
+        p, out, err = self.proc("genspider", file_name, "example.com")
+        self.assertIn(f"Created spider {file_name!r} using template 'basic' ", out)
         assert file_path.exists()
         modify_time_before = file_path.stat().st_mtime
         file_contents_before = file_path.read_text(encoding="utf-8")
 
         if force:
             # use different template to ensure contents were changed
-            p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
-            self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out)
+            p, out, err = self.proc(
+                "genspider", "--force", "-t", "crawl", file_name, "example.com"
+            )
+            self.assertIn(f"Created spider {file_name!r} using template 'crawl' ", out)
             modify_time_after = file_path.stat().st_mtime
             self.assertNotEqual(modify_time_after, modify_time_before)
             file_contents_after = file_path.read_text(encoding="utf-8")
             self.assertNotEqual(file_contents_after, file_contents_before)
         else:
-            p, out, err = self.proc('genspider', file_name, 'example.com')
-            self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out)
+            p, out, err = self.proc("genspider", file_name, "example.com")
+            self.assertIn(
+                f"{Path(self.temp_path, file_name + '.py').resolve()} already exists",
+                out,
+            )
             modify_time_after = file_path.stat().st_mtime
             self.assertEqual(modify_time_after, modify_time_before)
             file_contents_after = file_path.read_text(encoding="utf-8")
@@ -559,14 +600,13 @@ class GenspiderStandaloneCommandTest(ProjectTest):
 
 
 class MiscCommandsTest(CommandTest):
-
     def test_list(self):
-        self.assertEqual(0, self.call('list'))
+        self.assertEqual(0, self.call("list"))
 
 
 class RunSpiderCommandTest(CommandTest):
 
-    spider_filename = 'myspider.py'
+    spider_filename = "myspider.py"
 
     debug_log_spider = """
 import scrapy
@@ -604,7 +644,7 @@ class BadSpider(scrapy.Spider):
 
     def runspider(self, code, name=None, args=()):
         with self._create_file(code, name) as fname:
-            return self.proc('runspider', fname, *args)
+            return self.proc("runspider", fname, *args)
 
     def get_log(self, code, name=None, args=()):
         p, stdout, stderr = self.runspider(code, name, args=args)
@@ -618,18 +658,21 @@ class BadSpider(scrapy.Spider):
         self.assertIn("INFO: Spider closed (finished)", log)
 
     def test_run_fail_spider(self):
-        proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
+        proc, _, _ = self.runspider(
+            "import scrapy\n" + inspect.getsource(ExceptionSpider)
+        )
         ret = proc.returncode
         self.assertNotEqual(ret, 0)
 
     def test_run_good_spider(self):
-        proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
+        proc, _, _ = self.runspider(
+            "import scrapy\n" + inspect.getsource(NoRequestsSpider)
+        )
         ret = proc.returncode
         self.assertEqual(ret, 0)
 
     def test_runspider_log_level(self):
-        log = self.get_log(self.debug_log_spider,
-                           args=('-s', 'LOG_LEVEL=INFO'))
+        log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO"))
         self.assertNotIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
 
@@ -649,19 +692,17 @@ class MySpider(scrapy.Spider):
     def parse(self, response):
         return {'test': 'value'}
 """
-        log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
+        log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False"))
         self.assertNotIn("DNSLookupError", log)
         self.assertIn("INFO: Spider opened", log)
 
     def test_runspider_log_short_names(self):
-        log1 = self.get_log(self.debug_log_spider,
-                            args=('-s', 'LOG_SHORT_NAMES=1'))
+        log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1"))
         self.assertIn("[myspider] DEBUG: It Works!", log1)
         self.assertIn("[scrapy]", log1)
         self.assertNotIn("[scrapy.core.engine]", log1)
 
-        log2 = self.get_log(self.debug_log_spider,
-                            args=('-s', 'LOG_SHORT_NAMES=0'))
+        log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0"))
         self.assertIn("[myspider] DEBUG: It Works!", log2)
         self.assertNotIn("[scrapy]", log2)
         self.assertIn("[scrapy.core.engine]", log2)
@@ -671,57 +712,89 @@ class MySpider(scrapy.Spider):
         self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
-        _, _, log = self.proc('runspider', 'some_non_existent_file')
+        _, _, log = self.proc("runspider", "some_non_existent_file")
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
-        log = self.get_log('', name='myspider.txt')
-        self.assertIn('Unable to load', log)
+        log = self.get_log("", name="myspider.txt")
+        self.assertIn("Unable to load", log)
 
     def test_start_requests_errors(self):
-        log = self.get_log(self.badspider, name='badspider.py')
+        log = self.get_log(self.badspider, name="badspider.py")
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
     def test_asyncio_enabled_true(self):
-        log = self.get_log(self.debug_log_spider, args=[
-            '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
-        ])
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.get_log(
+            self.debug_log_spider,
+            args=[
+                "-s",
+                "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            ],
+        )
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_asyncio_enabled_default(self):
         log = self.get_log(self.debug_log_spider, args=[])
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_asyncio_enabled_false(self):
-        log = self.get_log(self.debug_log_spider, args=[
-            '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor'
-        ])
-        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
-        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.get_log(
+            self.debug_log_spider,
+            args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"],
+        )
+        self.assertIn(
+            "Using reactor: twisted.internet.selectreactor.SelectReactor", log
+        )
+        self.assertNotIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
-    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
-    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
-    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    @mark.skipif(
+        sys.implementation.name == "pypy",
+        reason="uvloop does not support pypy properly",
+    )
+    @mark.skipif(
+        platform.system() == "Windows", reason="uvloop does not support Windows"
+    )
+    @mark.skipif(
+        twisted_version == Version("twisted", 21, 2, 0),
+        reason="https://twistedmatrix.com/trac/ticket/10106",
+    )
     def test_custom_asyncio_loop_enabled_true(self):
-        log = self.get_log(self.debug_log_spider, args=[
-            '-s',
-            'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor',
-            '-s',
-            'ASYNCIO_EVENT_LOOP=uvloop.Loop',
-        ])
+        log = self.get_log(
+            self.debug_log_spider,
+            args=[
+                "-s",
+                "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                "-s",
+                "ASYNCIO_EVENT_LOOP=uvloop.Loop",
+            ],
+        )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
     def test_custom_asyncio_loop_enabled_false(self):
-        log = self.get_log(self.debug_log_spider, args=[
-            '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
-        ])
+        log = self.get_log(
+            self.debug_log_spider,
+            args=[
+                "-s",
+                "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            ],
+        )
         import asyncio
-        if sys.platform != 'win32':
+
+        if sys.platform != "win32":
             loop = asyncio.new_event_loop()
         else:
             loop = asyncio.SelectorEventLoop()
-        self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
+        self.assertIn(
+            f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}",
+            log,
+        )
 
     def test_output(self):
         spider_code = """
@@ -734,9 +807,11 @@ class MySpider(scrapy.Spider):
         self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
         return []
 """
-        args = ['-o', 'example.json']
+        args = ["-o", "example.json"]
         log = self.get_log(spider_code, args=args)
-        self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
+        self.assertIn(
+            "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log
+        )
 
     def test_overwrite_output(self):
         spider_code = """
@@ -755,9 +830,12 @@ class MySpider(scrapy.Spider):
         return []
 """
         Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8")
-        args = ['-O', 'example.json']
+        args = ["-O", "example.json"]
         log = self.get_log(spider_code, args=args)
-        self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+        self.assertIn(
+            '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}',
+            log,
+        )
         with Path(self.cwd, "example.json").open(encoding="utf-8") as f2:
             first_line = f2.readline()
         self.assertNotEqual(first_line, "not empty")
@@ -772,9 +850,11 @@ class MySpider(scrapy.Spider):
     def start_requests(self):
         return []
 """
-        args = ['-o', 'example1.json', '-O', 'example2.json']
+        args = ["-o", "example1.json", "-O", "example2.json"]
         log = self.get_log(spider_code, args=args)
-        self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+        self.assertIn(
+            "error: Please use only one of -o/--output and -O/--overwrite-output", log
+        )
 
     def test_output_stdout(self):
         spider_code = """
@@ -787,21 +867,21 @@ class MySpider(scrapy.Spider):
         self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
         return []
 """
-        args = ['-o', '-:json']
+        args = ["-o", "-:json"]
         log = self.get_log(spider_code, args=args)
         self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log)
 
 
-@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
 
-    spider_filename = 'myspider.pyw'
+    spider_filename = "myspider.pyw"
 
     def setUp(self):
         super().setUp()
 
     def test_start_requests_errors(self):
-        log = self.get_log(self.badspider, name='badspider.pyw')
+        log = self.get_log(self.badspider, name="badspider.pyw")
         self.assertIn("start_requests", log)
         self.assertIn("badspider.pyw", log)
 
@@ -834,34 +914,37 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
 
 
 class BenchCommandTest(CommandTest):
-
     def test_run(self):
-        _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
-                              '-s', 'CLOSESPIDER_TIMEOUT=0.01')
-        self.assertIn('INFO: Crawled', log)
-        self.assertNotIn('Unhandled Error', log)
+        _, _, log = self.proc(
+            "bench", "-s", "LOGSTATS_INTERVAL=0.001", "-s", "CLOSESPIDER_TIMEOUT=0.01"
+        )
+        self.assertIn("INFO: Crawled", log)
+        self.assertNotIn("Unhandled Error", log)
 
 
 class ViewCommandTest(CommandTest):
-
     def test_methods(self):
         command = view.Command()
         command.settings = Settings()
-        parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
-                                         formatter_class=ScrapyHelpFormatter,
-                                         conflict_handler='resolve')
+        parser = argparse.ArgumentParser(
+            prog="scrapy",
+            prefix_chars="-",
+            formatter_class=ScrapyHelpFormatter,
+            conflict_handler="resolve",
+        )
         command.add_options(parser)
-        self.assertEqual(command.short_desc(),
-                         "Open URL in browser, as seen by Scrapy")
-        self.assertIn("URL using the Scrapy downloader and show its",
-                      command.long_desc())
+        self.assertEqual(command.short_desc(), "Open URL in browser, as seen by Scrapy")
+        self.assertIn(
+            "URL using the Scrapy downloader and show its", command.long_desc()
+        )
 
 
 class CrawlCommandTest(CommandTest):
-
     def crawl(self, code, args=()):
-        Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code, encoding="utf-8")
-        return self.proc('crawl', 'myspider', *args)
+        Path(self.proj_mod_path, "spiders", "myspider.py").write_text(
+            code, encoding="utf-8"
+        )
+        return self.proc("crawl", "myspider", *args)
 
     def get_log(self, code, args=()):
         _, _, stderr = self.crawl(code, args=args)
@@ -892,9 +975,11 @@ class MySpider(scrapy.Spider):
         self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
         return []
 """
-        args = ['-o', 'example.json']
+        args = ["-o", "example.json"]
         log = self.get_log(spider_code, args=args)
-        self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
+        self.assertIn(
+            "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log
+        )
 
     def test_overwrite_output(self):
         spider_code = """
@@ -913,9 +998,12 @@ class MySpider(scrapy.Spider):
         return []
 """
         Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8")
-        args = ['-O', 'example.json']
+        args = ["-O", "example.json"]
         log = self.get_log(spider_code, args=args)
-        self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+        self.assertIn(
+            '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}',
+            log,
+        )
         with Path(self.cwd, "example.json").open(encoding="utf-8") as f2:
             first_line = f2.readline()
         self.assertNotEqual(first_line, "not empty")
@@ -930,18 +1018,32 @@ class MySpider(scrapy.Spider):
     def start_requests(self):
         return []
 """
-        args = ['-o', 'example1.json', '-O', 'example2.json']
+        args = ["-o", "example1.json", "-O", "example2.json"]
         log = self.get_log(spider_code, args=args)
-        self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+        self.assertIn(
+            "error: Please use only one of -o/--output and -O/--overwrite-output", log
+        )
 
 
 class HelpMessageTest(CommandTest):
-
     def setUp(self):
         super().setUp()
-        self.commands = ["parse", "startproject", "view", "crawl", "edit",
-                         "list", "fetch", "settings", "shell", "runspider",
-                         "version", "genspider", "check", "bench"]
+        self.commands = [
+            "parse",
+            "startproject",
+            "view",
+            "crawl",
+            "edit",
+            "list",
+            "fetch",
+            "settings",
+            "shell",
+            "runspider",
+            "version",
+            "genspider",
+            "check",
+            "bench",
+        ]
 
     def test_help_messages(self):
         for command in self.commands:
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 136056f50..7b104f618 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -26,52 +26,52 @@ class TestItem(Item):
 
 
 class ResponseMock:
-    url = 'http://scrapy.org'
+    url = "http://scrapy.org"
 
 
 class CustomSuccessContract(Contract):
-    name = 'custom_success_contract'
+    name = "custom_success_contract"
 
     def adjust_request_args(self, args):
-        args['url'] = 'http://scrapy.org'
+        args["url"] = "http://scrapy.org"
         return args
 
 
 class CustomFailContract(Contract):
-    name = 'custom_fail_contract'
+    name = "custom_fail_contract"
 
     def adjust_request_args(self, args):
-        raise TypeError('Error in adjust_request_args')
+        raise TypeError("Error in adjust_request_args")
 
 
 class CustomFormContract(Contract):
-    name = 'custom_form'
+    name = "custom_form"
     request_cls = FormRequest
 
     def adjust_request_args(self, args):
-        args['formdata'] = {'name': 'scrapy'}
+        args["formdata"] = {"name": "scrapy"}
         return args
 
 
 class TestSpider(Spider):
-    name = 'demo_spider'
+    name = "demo_spider"
 
     def returns_request(self, response):
-        """ method which returns request
+        """method which returns request
         @url http://scrapy.org
         @returns requests 1
         """
-        return Request('http://scrapy.org', callback=self.returns_item)
+        return Request("http://scrapy.org", callback=self.returns_item)
 
     def returns_item(self, response):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @returns items 1 1
         """
         return TestItem(url=response.url)
 
     def returns_request_cb_kwargs(self, response, url):
-        """ method which returns request
+        """method which returns request
         @url https://example.org
         @cb_kwargs {"url": "http://scrapy.org"}
         @returns requests 1
@@ -79,7 +79,7 @@ class TestSpider(Spider):
         return Request(url, callback=self.returns_item_cb_kwargs)
 
     def returns_item_cb_kwargs(self, response, name):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @cb_kwargs {"name": "Scrapy"}
         @returns items 1 1
@@ -87,7 +87,7 @@ class TestSpider(Spider):
         return TestItem(name=name, url=response.url)
 
     def returns_item_cb_kwargs_error_unexpected_keyword(self, response):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @cb_kwargs {"arg": "value"}
         @returns items 1 1
@@ -95,51 +95,51 @@ class TestSpider(Spider):
         return TestItem(url=response.url)
 
     def returns_item_cb_kwargs_error_missing_argument(self, response, arg):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @returns items 1 1
         """
         return TestItem(url=response.url)
 
     def returns_dict_item(self, response):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @returns items 1 1
         """
         return {"url": response.url}
 
     def returns_fail(self, response):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @returns items 0 0
         """
         return TestItem(url=response.url)
 
     def returns_dict_fail(self, response):
-        """ method which returns item
+        """method which returns item
         @url http://scrapy.org
         @returns items 0 0
         """
-        return {'url': response.url}
+        return {"url": response.url}
 
     def scrapes_item_ok(self, response):
-        """ returns item with name and url
+        """returns item with name and url
         @url http://scrapy.org
         @returns items 1 1
         @scrapes name url
         """
-        return TestItem(name='test', url=response.url)
+        return TestItem(name="test", url=response.url)
 
     def scrapes_dict_item_ok(self, response):
-        """ returns item with name and url
+        """returns item with name and url
         @url http://scrapy.org
         @returns items 1 1
         @scrapes name url
         """
-        return {'name': 'test', 'url': response.url}
+        return {"name": "test", "url": response.url}
 
     def scrapes_item_fail(self, response):
-        """ returns item with no name
+        """returns item with no name
         @url http://scrapy.org
         @returns items 1 1
         @scrapes name url
@@ -147,15 +147,15 @@ class TestSpider(Spider):
         return TestItem(url=response.url)
 
     def scrapes_dict_item_fail(self, response):
-        """ returns item with no name
+        """returns item with no name
         @url http://scrapy.org
         @returns items 1 1
         @scrapes name url
         """
-        return {'url': response.url}
+        return {"url": response.url}
 
     def scrapes_multiple_missing_fields(self, response):
-        """ returns item with no name
+        """returns item with no name
         @url http://scrapy.org
         @returns items 1 1
         @scrapes name url
@@ -163,7 +163,7 @@ class TestSpider(Spider):
         return {}
 
     def parse_no_url(self, response):
-        """ method with no url
+        """method with no url
         @returns items 1 1
         """
         pass
@@ -177,7 +177,7 @@ class TestSpider(Spider):
 
 
 class CustomContractSuccessSpider(Spider):
-    name = 'custom_contract_success_spider'
+    name = "custom_contract_success_spider"
 
     def parse(self, response):
         """
@@ -187,7 +187,7 @@ class CustomContractSuccessSpider(Spider):
 
 
 class CustomContractFailSpider(Spider):
-    name = 'custom_contract_fail_spider'
+    name = "custom_contract_fail_spider"
 
     def parse(self, response):
         """
@@ -197,7 +197,7 @@ class CustomContractFailSpider(Spider):
 
 
 class InheritsTestSpider(TestSpider):
-    name = 'inherits_demo_spider'
+    name = "inherits_demo_spider"
 
 
 class ContractsManagerTest(unittest.TestCase):
@@ -234,7 +234,8 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertEqual(len(contracts), 2)
         self.assertEqual(
             frozenset(type(x) for x in contracts),
-            frozenset([UrlContract, ReturnsContract]))
+            frozenset([UrlContract, ReturnsContract]),
+        )
 
         # returns request for valid method
         request = self.conman.from_method(spider.returns_request, self.results)
@@ -251,26 +252,40 @@ class ContractsManagerTest(unittest.TestCase):
         # extract contracts correctly
         contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs)
         self.assertEqual(len(contracts), 3)
-        self.assertEqual(frozenset(type(x) for x in contracts),
-                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]),
+        )
 
         contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
         self.assertEqual(len(contracts), 3)
-        self.assertEqual(frozenset(type(x) for x in contracts),
-                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]),
+        )
 
-        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword)
+        contracts = self.conman.extract_contracts(
+            spider.returns_item_cb_kwargs_error_unexpected_keyword
+        )
         self.assertEqual(len(contracts), 3)
-        self.assertEqual(frozenset(type(x) for x in contracts),
-                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]),
+        )
 
-        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument)
+        contracts = self.conman.extract_contracts(
+            spider.returns_item_cb_kwargs_error_missing_argument
+        )
         self.assertEqual(len(contracts), 2)
-        self.assertEqual(frozenset(type(x) for x in contracts),
-                         frozenset([UrlContract, ReturnsContract]))
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, ReturnsContract]),
+        )
 
         # returns_request
-        request = self.conman.from_method(spider.returns_request_cb_kwargs, self.results)
+        request = self.conman.from_method(
+            spider.returns_request_cb_kwargs, self.results
+        )
         request.callback(response, **request.cb_kwargs)
         self.should_succeed()
 
@@ -280,12 +295,16 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_succeed()
 
         # returns_item (error, callback doesn't take keyword arguments)
-        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results)
+        request = self.conman.from_method(
+            spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results
+        )
         request.callback(response, **request.cb_kwargs)
         self.should_error()
 
         # returns_item (error, contract doesn't provide keyword arguments)
-        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_missing_argument, self.results)
+        request = self.conman.from_method(
+            spider.returns_item_cb_kwargs_error_missing_argument, self.results
+        )
         request.callback(response, **request.cb_kwargs)
         self.should_error()
 
@@ -343,10 +362,12 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_fail()
 
         # scrapes_multiple_missing_fields
-        request = self.conman.from_method(spider.scrapes_multiple_missing_fields, self.results)
+        request = self.conman.from_method(
+            spider.scrapes_multiple_missing_fields, self.results
+        )
         request.callback(response)
         self.should_fail()
-        message = 'ContractFail: Missing fields: name, url'
+        message = "ContractFail: Missing fields: name, url"
         assert message in self.results.failures[-1][-1]
 
     def test_custom_contracts(self):
@@ -361,7 +382,7 @@ class ContractsManagerTest(unittest.TestCase):
         response = ResponseMock()
 
         try:
-            raise HttpError(response, 'Ignoring non-200 response')
+            raise HttpError(response, "Ignoring non-200 response")
         except HttpError:
             failure_mock = failure.Failure()
 
@@ -373,9 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_same_url(self):
-
         class TestSameUrlSpider(Spider):
-            name = 'test_same_url'
+            name = "test_same_url"
 
             def __init__(self, *args, **kwargs):
                 super().__init__(*args, **kwargs)
@@ -406,7 +426,7 @@ class ContractsManagerTest(unittest.TestCase):
     def test_form_contract(self):
         spider = TestSpider()
         request = self.conman.from_method(spider.custom_form, self.results)
-        self.assertEqual(request.method, 'POST')
+        self.assertEqual(request.method, "POST")
         self.assertIsInstance(request, FormRequest)
 
     def test_inherited_contracts(self):
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 113ea8f19..9a6e9e4ff 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -4,7 +4,8 @@ from scrapy.core.downloader import Slot
 
 
 class SlotTest(unittest.TestCase):
-
     def test_repr(self):
         slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
-        self.assertEqual(repr(slot), 'Slot(concurrency=8, delay=0.10, randomize_delay=True)')
+        self.assertEqual(
+            repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)"
+        )
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 5ec96e4a7..3f19bef22 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -53,7 +53,6 @@ from tests.spiders import (
 
 
 class CrawlTestCase(TestCase):
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -82,17 +81,17 @@ class CrawlTestCase(TestCase):
             mockserver=self.mockserver,
             total=total,
         )
-        tolerance = (1 - (0.6 if randomize else 0.2))
+        tolerance = 1 - (0.6 if randomize else 0.2)
 
-        settings = {"DOWNLOAD_DELAY": delay,
-                    'RANDOMIZE_DOWNLOAD_DELAY': randomize}
+        settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize}
         crawler = get_crawler(FollowAllSpider, settings)
         yield crawler.crawl(**crawl_kwargs)
         times = crawler.spider.times
         total_time = times[-1] - times[0]
         average = total_time / (len(times) - 1)
-        self.assertTrue(average > delay * tolerance,
-                        f"download delay too small: {average}")
+        self.assertTrue(
+            average > delay * tolerance, f"download delay too small: {average}"
+        )
 
         # Ensure that the same test parameters would cause a failure if no
         # download delay is set. Otherwise, it means we are using a combination
@@ -104,8 +103,9 @@ class CrawlTestCase(TestCase):
         times = crawler.spider.times
         total_time = times[-1] - times[0]
         average = total_time / (len(times) - 1)
-        self.assertFalse(average > delay / tolerance,
-                         "test total or delay values are too small")
+        self.assertFalse(
+            average > delay / tolerance, "test total or delay values are too small"
+        )
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
@@ -134,14 +134,18 @@ class CrawlTestCase(TestCase):
     def test_retry_503(self):
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/status?n=503"), mockserver=self.mockserver
+            )
         self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
+            yield crawler.crawl(
+                "http://localhost:65432/status?n=503", mockserver=self.mockserver
+            )
         self._assert_retried(log)
 
     @defer.inlineCallbacks
@@ -151,12 +155,14 @@ class CrawlTestCase(TestCase):
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             # try to fetch the homepage of a non-existent domain
-            yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
+            yield crawler.crawl(
+                "http://dns.resolution.invalid./", mockserver=self.mockserver
+            )
         self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
-        with LogCapture('scrapy', level=logging.ERROR) as log:
+        with LogCapture("scrapy", level=logging.ERROR) as log:
             crawler = get_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
 
@@ -167,7 +173,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
-        with LogCapture('scrapy', level=logging.ERROR) as log:
+        with LogCapture("scrapy", level=logging.ERROR) as log:
             crawler = get_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
 
@@ -183,16 +189,24 @@ class CrawlTestCase(TestCase):
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(
             crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
-            crawler.spider.seedsseen)
+            crawler.spider.seedsseen,
+        )
 
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = get_crawler(DuplicateStartRequestsSpider, settings)
-        yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
+        yield crawler.crawl(
+            dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver
+        )
         self.assertEqual(crawler.spider.visited, 6)
 
-        yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver)
+        yield crawler.crawl(
+            dont_filter=False,
+            distinct_urls=3,
+            dupe_factor=4,
+            mockserver=self.mockserver,
+        )
         self.assertEqual(crawler.spider.visited, 3)
 
     @defer.inlineCallbacks
@@ -200,7 +214,10 @@ class CrawlTestCase(TestCase):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
         from urllib.parse import urlencode
-        query = urlencode({'raw': '''\
+
+        query = urlencode(
+            {
+                "raw": """\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1
 X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0
@@ -216,10 +233,14 @@ Connection: close
 
 foo body
 with multiples lines
-'''})
+"""
+            }
+        )
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver
+            )
         self.assertEqual(str(log).count("Got response 200"), 1)
 
     @defer.inlineCallbacks
@@ -227,7 +248,9 @@ with multiples lines
         # connection lost after receiving data
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver
+            )
         self._assert_retried(log)
 
     @defer.inlineCallbacks
@@ -235,7 +258,9 @@ with multiples lines
         # connection lost before receiving data
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver
+            )
         self._assert_retried(log)
 
     def _assert_retried(self, log):
@@ -245,56 +270,62 @@ with multiples lines
     @defer.inlineCallbacks
     def test_referer_header(self):
         """Referer header is set by RefererMiddleware unless it is already set"""
-        req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1)
+        req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1)
         req1 = req0.replace()
-        req2 = req0.replace(headers={'Referer': None})
-        req3 = req0.replace(headers={'Referer': 'http://example.com'})
-        req0.meta['next'] = req1
-        req1.meta['next'] = req2
-        req2.meta['next'] = req3
+        req2 = req0.replace(headers={"Referer": None})
+        req3 = req0.replace(headers={"Referer": "http://example.com"})
+        req0.meta["next"] = req1
+        req1.meta["next"] = req2
+        req2.meta["next"] = req3
         crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=req0, mockserver=self.mockserver)
         # basic asserts in case of weird communication errors
-        self.assertIn('responses', crawler.spider.meta)
-        self.assertNotIn('failures', crawler.spider.meta)
+        self.assertIn("responses", crawler.spider.meta)
+        self.assertNotIn("failures", crawler.spider.meta)
         # start requests doesn't set Referer header
-        echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
-        self.assertNotIn('Referer', echo0['headers'])
+        echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
+        self.assertNotIn("Referer", echo0["headers"])
         # following request sets Referer to start request url
-        echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body))
-        self.assertEqual(echo1['headers'].get('Referer'), [req0.url])
+        echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body))
+        self.assertEqual(echo1["headers"].get("Referer"), [req0.url])
         # next request avoids Referer header
-        echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
-        self.assertNotIn('Referer', echo2['headers'])
+        echo2 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
+        self.assertNotIn("Referer", echo2["headers"])
         # last request explicitly sets a Referer header
-        echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body))
-        self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
+        echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body))
+        self.assertEqual(echo3["headers"].get("Referer"), ["http://example.com"])
 
     @defer.inlineCallbacks
     def test_engine_status(self):
         from scrapy.utils.engine import get_engine_status
+
         est = []
 
         def cb(response):
             est.append(get_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
+        yield crawler.crawl(
+            seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver
+        )
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
-        self.assertEqual(s['engine.spider.name'], crawler.spider.name)
-        self.assertEqual(s['len(engine.scraper.slot.active)'], 1)
+        self.assertEqual(s["engine.spider.name"], crawler.spider.name)
+        self.assertEqual(s["len(engine.scraper.slot.active)"], 1)
 
     @defer.inlineCallbacks
     def test_format_engine_status(self):
         from scrapy.utils.engine import format_engine_status
+
         est = []
 
         def cb(response):
             est.append(format_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
+        yield crawler.crawl(
+            seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver
+        )
         self.assertEqual(len(est), 1, est)
         est = est[0].split("\n")[2:-2]  # remove header & footer
         # convert to dict
@@ -304,8 +335,8 @@ with multiples lines
         it = iter(est)
         s = dict(zip(it, it))
 
-        self.assertEqual(s['engine.spider.name'], crawler.spider.name)
-        self.assertEqual(s['len(engine.scraper.slot.active)'], '1')
+        self.assertEqual(s["engine.spider.name"], crawler.spider.name)
+        self.assertEqual(s["len(engine.scraper.slot.active)"], "1")
 
     @defer.inlineCallbacks
     def test_graceful_crawl_error_handling(self):
@@ -337,8 +368,11 @@ with multiples lines
         }
         crawler = get_crawler(SimpleSpider, settings)
         yield self.assertFailure(
-            crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
-            ZeroDivisionError)
+            crawler.crawl(
+                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+            ),
+            ZeroDivisionError,
+        )
         self.assertFalse(crawler.crawling)
 
     @defer.inlineCallbacks
@@ -346,14 +380,26 @@ with multiples lines
         crawler = get_crawler(SimpleSpider)
         runner = CrawlerRunner()
         with LogCapture() as log:
-            yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield runner.crawl(
+                crawler,
+                self.mockserver.url("/status?n=200"),
+                mockserver=self.mockserver,
+            )
         self.assertIn("Got response 200", str(log))
 
     @defer.inlineCallbacks
     def test_crawl_multiple(self):
-        runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
-        runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
-        runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
+        runner = CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
+        runner.crawl(
+            SimpleSpider,
+            self.mockserver.url("/status?n=200"),
+            mockserver=self.mockserver,
+        )
+        runner.crawl(
+            SimpleSpider,
+            self.mockserver.url("/status?n=503"),
+            mockserver=self.mockserver,
+        )
 
         with LogCapture() as log:
             yield runner.join()
@@ -363,7 +409,6 @@ with multiples lines
 
 
 class CrawlSpiderTestCase(TestCase):
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -381,7 +426,9 @@ class CrawlSpiderTestCase(TestCase):
         crawler = get_crawler(spider_cls)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+            )
         return log, items, crawler.stats
 
     @defer.inlineCallbacks
@@ -441,17 +488,24 @@ class CrawlSpiderTestCase(TestCase):
     def test_async_def_parse(self):
         crawler = get_crawler(AsyncDefSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+            )
         self.assertIn("Got response 200", str(log))
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse(self):
-        crawler = get_crawler(AsyncDefAsyncioSpider, {
-            "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
-        })
+        crawler = get_crawler(
+            AsyncDefAsyncioSpider,
+            {
+                "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+            },
+        )
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+            )
         self.assertIn("Got response 200", str(log))
 
     @mark.only_asyncio()
@@ -459,8 +513,8 @@ class CrawlSpiderTestCase(TestCase):
     def test_async_def_asyncio_parse_items_list(self):
         log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
         self.assertIn("Got response 200", str(log))
-        self.assertIn({'id': 1}, items)
-        self.assertIn({'id': 2}, items)
+        self.assertIn({"id": 1}, items)
+        self.assertIn({"id": 2}, items)
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
@@ -473,7 +527,9 @@ class CrawlSpiderTestCase(TestCase):
         crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield crawler.crawl(
+                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+            )
         self.assertIn("Got response 200", str(log))
         self.assertIn({"foo": 42}, items)
 
@@ -482,7 +538,7 @@ class CrawlSpiderTestCase(TestCase):
     def test_async_def_asyncgen_parse(self):
         log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
         self.assertIn("Got response 200", str(log))
-        itemcount = stats.get_value('item_scraped_count')
+        itemcount = stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, 1)
 
     @mark.only_asyncio()
@@ -490,10 +546,10 @@ class CrawlSpiderTestCase(TestCase):
     def test_async_def_asyncgen_parse_loop(self):
         log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
         self.assertIn("Got response 200", str(log))
-        itemcount = stats.get_value('item_scraped_count')
+        itemcount = stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, 10)
         for i in range(10):
-            self.assertIn({'foo': i}, items)
+            self.assertIn({"foo": i}, items)
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
@@ -502,22 +558,22 @@ class CrawlSpiderTestCase(TestCase):
         log = str(log)
         self.assertIn("Spider error processing", log)
         self.assertIn("ValueError", log)
-        itemcount = stats.get_value('item_scraped_count')
+        itemcount = stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, 7)
         for i in range(7):
-            self.assertIn({'foo': i}, items)
+            self.assertIn({"foo": i}, items)
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_complex(self):
         _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
-        itemcount = stats.get_value('item_scraped_count')
+        itemcount = stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, 156)
         # some random items
         for i in [1, 4, 21, 22, 207, 311]:
-            self.assertIn({'index': i}, items)
+            self.assertIn({"index": i}, items)
         for i in [10, 30, 122]:
-            self.assertIn({'index2': i}, items)
+            self.assertIn({"index2": i}, items)
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
@@ -530,32 +586,32 @@ class CrawlSpiderTestCase(TestCase):
     @defer.inlineCallbacks
     def test_async_def_deferred_direct(self):
         _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
-        self.assertEqual(items, [{'code': 200}])
+        self.assertEqual(items, [{"code": 200}])
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_deferred_wrapped(self):
         log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
-        self.assertEqual(items, [{'code': 200}])
+        self.assertEqual(items, [{"code": 200}])
 
     @defer.inlineCallbacks
     def test_async_def_deferred_maybe_wrapped(self):
         _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
-        self.assertEqual(items, [{'code': 200}])
+        self.assertEqual(items, [{"code": 200}])
 
     @defer.inlineCallbacks
     def test_response_ssl_certificate_none(self):
         crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/echo?body=test", is_secure=False)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
-        self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
+        self.assertIsNone(crawler.spider.meta["responses"][0].certificate)
 
     @defer.inlineCallbacks
     def test_response_ssl_certificate(self):
         crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/echo?body=test", is_secure=True)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
-        cert = crawler.spider.meta['responses'][0].certificate
+        cert = crawler.spider.meta["responses"][0].certificate
         self.assertIsInstance(cert, Certificate)
         self.assertEqual(cert.getSubject().commonName, b"localhost")
         self.assertEqual(cert.getIssuer().commonName, b"localhost")
@@ -566,7 +622,7 @@ class CrawlSpiderTestCase(TestCase):
         crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/status?n=200", is_secure=True)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
-        cert = crawler.spider.meta['responses'][0].certificate
+        cert = crawler.spider.meta["responses"][0].certificate
         self.assertIsInstance(cert, Certificate)
         self.assertEqual(cert.getSubject().commonName, b"localhost")
         self.assertEqual(cert.getIssuer().commonName, b"localhost")
@@ -574,18 +630,18 @@ class CrawlSpiderTestCase(TestCase):
     @defer.inlineCallbacks
     def test_dns_server_ip_address_none(self):
         crawler = get_crawler(SingleRequestSpider)
-        url = self.mockserver.url('/status?n=200')
+        url = self.mockserver.url("/status?n=200")
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
-        ip_address = crawler.spider.meta['responses'][0].ip_address
+        ip_address = crawler.spider.meta["responses"][0].ip_address
         self.assertIsNone(ip_address)
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address(self):
         crawler = get_crawler(SingleRequestSpider)
-        url = self.mockserver.url('/echo?body=test')
-        expected_netloc, _ = urlparse(url).netloc.split(':')
+        url = self.mockserver.url("/echo?body=test")
+        expected_netloc, _ = urlparse(url).netloc.split(":")
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
-        ip_address = crawler.spider.meta['responses'][0].ip_address
+        ip_address = crawler.spider.meta["responses"][0].ip_address
         self.assertIsInstance(ip_address, IPv4Address)
         self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
 
@@ -595,8 +651,14 @@ class CrawlSpiderTestCase(TestCase):
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
-        self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received"))
-        self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
+        self.assertEqual(
+            crawler.spider.meta["response"].body,
+            crawler.spider.meta.get("bytes_received"),
+        )
+        self.assertLess(
+            len(crawler.spider.meta["response"].body),
+            crawler.spider.full_response_length,
+        )
 
     @defer.inlineCallbacks
     def test_bytes_received_stop_download_errback(self):
@@ -608,10 +670,12 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
         self.assertEqual(
             crawler.spider.meta["failure"].value.response.body,
-            crawler.spider.meta.get("bytes_received"))
+            crawler.spider.meta.get("bytes_received"),
+        )
         self.assertLess(
             len(crawler.spider.meta["failure"].value.response.body),
-            crawler.spider.full_response_length)
+            crawler.spider.full_response_length,
+        )
 
     @defer.inlineCallbacks
     def test_headers_received_stop_download_callback(self):
@@ -619,7 +683,10 @@ class CrawlSpiderTestCase(TestCase):
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
-        self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received"))
+        self.assertEqual(
+            crawler.spider.meta["response"].headers,
+            crawler.spider.meta.get("headers_received"),
+        )
 
     @defer.inlineCallbacks
     def test_headers_received_stop_download_errback(self):
@@ -631,4 +698,5 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
         self.assertEqual(
             crawler.spider.meta["failure"].value.response.headers,
-            crawler.spider.meta.get("headers_received"))
+            crawler.spider.meta.get("headers_received"),
+        )
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index e0902fdbe..c6b93599e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -31,44 +31,42 @@ from tests.mockserver import MockServer
 
 
 class BaseCrawlerTest(unittest.TestCase):
-
     def assertOptionIsDefault(self, settings, key):
         self.assertIsInstance(settings, Settings)
         self.assertEqual(settings[key], getattr(default_settings, key))
 
 
 class CrawlerTestCase(BaseCrawlerTest):
-
     def test_populate_spidercls_settings(self):
-        spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
-        project_settings = {'TEST1': 'project', 'TEST3': 'project'}
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {"TEST1": "project", "TEST3": "project"}
 
         class CustomSettingsSpider(DefaultSpider):
             custom_settings = spider_settings
 
         settings = Settings()
-        settings.setdict(project_settings, priority='project')
+        settings.setdict(project_settings, priority="project")
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
             crawler = Crawler(CustomSettingsSpider, settings)
 
-        self.assertEqual(crawler.settings.get('TEST1'), 'spider')
-        self.assertEqual(crawler.settings.get('TEST2'), 'spider')
-        self.assertEqual(crawler.settings.get('TEST3'), 'project')
+        self.assertEqual(crawler.settings.get("TEST1"), "spider")
+        self.assertEqual(crawler.settings.get("TEST2"), "spider")
+        self.assertEqual(crawler.settings.get("TEST3"), "project")
 
         self.assertFalse(settings.frozen)
         self.assertTrue(crawler.settings.frozen)
 
     def test_crawler_accepts_dict(self):
-        crawler = get_crawler(DefaultSpider, {'foo': 'bar'})
-        self.assertEqual(crawler.settings['foo'], 'bar')
-        self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
+        crawler = get_crawler(DefaultSpider, {"foo": "bar"})
+        self.assertEqual(crawler.settings["foo"], "bar")
+        self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_accepts_None(self):
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
             crawler = Crawler(DefaultSpider)
-        self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
+        self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_rejects_spider_objects(self):
         with raises(ValueError):
@@ -78,10 +76,8 @@ class CrawlerTestCase(BaseCrawlerTest):
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):
         class MySpider(scrapy.Spider):
-            name = 'spider'
-            custom_settings = {
-                'AUTOTHROTTLE_ENABLED': True
-            }
+            name = "spider"
+            custom_settings = {"AUTOTHROTTLE_ENABLED": True}
 
         crawler = get_crawler(MySpider)
         enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
@@ -95,73 +91,71 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             logging.root.removeHandler(handler)
 
         class MySpider(scrapy.Spider):
-            name = 'spider'
+            name = "spider"
 
         get_crawler(MySpider)
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
         log_file = Path(self.mktemp())
-        log_file.write_text('previous message\n', encoding='utf-8')
+        log_file.write_text("previous message\n", encoding="utf-8")
 
         class MySpider(scrapy.Spider):
-            name = 'spider'
+            name = "spider"
             custom_settings = {
-                'LOG_LEVEL': 'INFO',
-                'LOG_FILE': str(log_file),
+                "LOG_LEVEL": "INFO",
+                "LOG_FILE": str(log_file),
                 # settings to avoid extra warnings
-                'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
-                'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+                "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
             }
 
         configure_logging()
         self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
         crawler = get_crawler(MySpider)
         self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
-        info_count = crawler.stats.get_value('log_count/INFO')
-        logging.debug('debug message')
-        logging.info('info message')
-        logging.warning('warning message')
-        logging.error('error message')
+        info_count = crawler.stats.get_value("log_count/INFO")
+        logging.debug("debug message")
+        logging.info("info message")
+        logging.warning("warning message")
+        logging.error("error message")
 
-        logged = log_file.read_text(encoding='utf-8')
+        logged = log_file.read_text(encoding="utf-8")
 
-        self.assertIn('previous message', logged)
-        self.assertNotIn('debug message', logged)
-        self.assertIn('info message', logged)
-        self.assertIn('warning message', logged)
-        self.assertIn('error message', logged)
-        self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1)
-        self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1)
-        self.assertEqual(
-            crawler.stats.get_value('log_count/INFO') - info_count, 1)
-        self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
+        self.assertIn("previous message", logged)
+        self.assertNotIn("debug message", logged)
+        self.assertIn("info message", logged)
+        self.assertIn("warning message", logged)
+        self.assertIn("error message", logged)
+        self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1)
+        self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1)
+        self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1)
+        self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0)
 
     def test_spider_custom_settings_log_append(self):
         log_file = Path(self.mktemp())
-        log_file.write_text('previous message\n', encoding='utf-8')
+        log_file.write_text("previous message\n", encoding="utf-8")
 
         class MySpider(scrapy.Spider):
-            name = 'spider'
+            name = "spider"
             custom_settings = {
-                'LOG_FILE': str(log_file),
-                'LOG_FILE_APPEND': False,
+                "LOG_FILE": str(log_file),
+                "LOG_FILE_APPEND": False,
                 # disable telnet if not available to avoid an extra warning
-                'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+                "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
             }
 
         configure_logging()
         get_crawler(MySpider)
-        logging.debug('debug message')
+        logging.debug("debug message")
 
-        logged = log_file.read_text(encoding='utf-8')
+        logged = log_file.read_text(encoding="utf-8")
 
-        self.assertNotIn('previous message', logged)
-        self.assertIn('debug message', logged)
+        self.assertNotIn("previous message", logged)
+        self.assertIn("debug message", logged)
 
 
 class SpiderLoaderWithWrongInterface:
-
     def unneeded_method(self):
         pass
 
@@ -171,11 +165,12 @@ class CustomSpiderLoader(SpiderLoader):
 
 
 class CrawlerRunnerTestCase(BaseCrawlerTest):
-
     def test_spider_manager_verify_interface(self):
-        settings = Settings({
-            'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface,
-        })
+        settings = Settings(
+            {
+                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
+            }
+        )
         with warnings.catch_warnings(record=True) as w:
             self.assertRaises(AttributeError, CrawlerRunner, settings)
             self.assertEqual(len(w), 1)
@@ -183,13 +178,13 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
             self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
 
     def test_crawler_runner_accepts_dict(self):
-        runner = CrawlerRunner({'foo': 'bar'})
-        self.assertEqual(runner.settings['foo'], 'bar')
-        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+        runner = CrawlerRunner({"foo": "bar"})
+        self.assertEqual(runner.settings["foo"], "bar")
+        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
     def test_crawler_runner_accepts_None(self):
         runner = CrawlerRunner()
-        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
     def test_deprecated_attribute_spiders(self):
         with warnings.catch_warnings(record=True) as w:
@@ -198,41 +193,40 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
             self.assertEqual(len(w), 1)
             self.assertIn("CrawlerRunner.spiders", str(w[0].message))
             self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
-            sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
+            sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"])
             self.assertIsInstance(spiders, sl_cls)
 
 
 class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):
-        runner = CrawlerProcess({'foo': 'bar'})
-        self.assertEqual(runner.settings['foo'], 'bar')
-        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+        runner = CrawlerProcess({"foo": "bar"})
+        self.assertEqual(runner.settings["foo"], "bar")
+        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
     def test_crawler_process_accepts_None(self):
         runner = CrawlerProcess()
-        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
 
 class ExceptionSpider(scrapy.Spider):
-    name = 'exception'
+    name = "exception"
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
-        raise ValueError('Exception in from_crawler method')
+        raise ValueError("Exception in from_crawler method")
 
 
 class NoRequestsSpider(scrapy.Spider):
-    name = 'no_request'
+    name = "no_request"
 
     def start_requests(self):
         return []
 
 
-@mark.usefixtures('reactor_pytest')
+@mark.usefixtures("reactor_pytest")
 class CrawlerRunnerHasSpider(unittest.TestCase):
-
     def _runner(self):
-        return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
+        return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
@@ -256,7 +250,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         except ValueError:
             pass
         else:
-            self.fail('Exception should be raised from spider')
+            self.fail("Exception should be raised from spider")
 
         self.assertEqual(runner.bootstrap_failed, True)
 
@@ -269,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         except ValueError:
             pass
         else:
-            self.fail('Exception should be raised from spider')
+            self.fail("Exception should be raised from spider")
 
         yield runner.crawl(NoRequestsSpider)
 
@@ -277,18 +271,22 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler_runner_asyncio_enabled_true(self):
-        if self.reactor_pytest == 'asyncio':
-            CrawlerRunner(settings={
-                "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
-            })
+        if self.reactor_pytest == "asyncio":
+            CrawlerRunner(
+                settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+                }
+            )
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                runner = CrawlerRunner(settings={
-                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
-                })
+                runner = CrawlerRunner(
+                    settings={
+                        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                        "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+                    }
+                )
                 yield runner.crawl(NoRequestsSpider)
 
 
@@ -298,35 +296,42 @@ class ScriptRunnerMixin:
     def run_script(self, script_name: str, *script_args):
         script_path = self.script_dir / script_name
         args = [sys.executable, str(script_path)] + list(script_args)
-        p = subprocess.Popen(args, env=get_testenv(),
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen(
+            args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
         stdout, stderr = p.communicate()
-        return stderr.decode('utf-8')
+        return stderr.decode("utf-8")
 
 
 class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
-    script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess'
+    script_dir = Path(__file__).parent.resolve() / "CrawlerProcess"
 
     def test_simple(self):
-        log = self.run_script('simple.py')
-        self.assertIn('Spider closed (finished)', log)
-        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.run_script("simple.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertNotIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_multi(self):
-        log = self.run_script('multi.py')
-        self.assertIn('Spider closed (finished)', log)
-        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.run_script("multi.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertNotIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
         self.assertNotIn("ReactorAlreadyInstalledError", log)
 
     def test_reactor_default(self):
-        log = self.run_script('reactor_default.py')
-        self.assertIn('Spider closed (finished)', log)
-        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.run_script("reactor_default.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertNotIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
         self.assertNotIn("ReactorAlreadyInstalledError", log)
 
     def test_reactor_default_twisted_reactor_select(self):
-        log = self.run_script('reactor_default_twisted_reactor_select.py')
-        if platform.system() in ['Windows', 'Darwin']:
+        log = self.run_script("reactor_default_twisted_reactor_select.py")
+        if platform.system() in ["Windows", "Darwin"]:
             # The goal of this test function is to test that, when a reactor is
             # installed (the default one here) and a different reactor is
             # configured (select here), an error raises.
@@ -337,9 +342,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             # If that ever becomes the case on more platforms (i.e. if Linux
             # also starts using the select reactor by default in a future
             # version of Twisted), then we will need to rethink this test.
-            self.assertIn('Spider closed (finished)', log)
+            self.assertIn("Spider closed (finished)", log)
         else:
-            self.assertNotIn('Spider closed (finished)', log)
+            self.assertNotIn("Spider closed (finished)", log)
             self.assertIn(
                 (
                     "does not match the requested one "
@@ -349,18 +354,18 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             )
 
     def test_reactor_select(self):
-        log = self.run_script('reactor_select.py')
-        self.assertIn('Spider closed (finished)', log)
+        log = self.run_script("reactor_select.py")
+        self.assertIn("Spider closed (finished)", log)
         self.assertNotIn("ReactorAlreadyInstalledError", log)
 
     def test_reactor_select_twisted_reactor_select(self):
-        log = self.run_script('reactor_select_twisted_reactor_select.py')
-        self.assertIn('Spider closed (finished)', log)
+        log = self.run_script("reactor_select_twisted_reactor_select.py")
+        self.assertIn("Spider closed (finished)", log)
         self.assertNotIn("ReactorAlreadyInstalledError", log)
 
     def test_reactor_select_subclass_twisted_reactor_select(self):
-        log = self.run_script('reactor_select_subclass_twisted_reactor_select.py')
-        self.assertNotIn('Spider closed (finished)', log)
+        log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")
+        self.assertNotIn("Spider closed (finished)", log)
         self.assertIn(
             (
                 "does not match the requested one "
@@ -370,24 +375,34 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         )
 
     def test_asyncio_enabled_no_reactor(self):
-        log = self.run_script('asyncio_enabled_no_reactor.py')
-        self.assertIn('Spider closed (finished)', log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.run_script("asyncio_enabled_no_reactor.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_asyncio_enabled_reactor(self):
-        log = self.run_script('asyncio_enabled_reactor.py')
-        self.assertIn('Spider closed (finished)', log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        log = self.run_script("asyncio_enabled_reactor.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
-    @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"),
-                 reason='w3lib 2.0.0 and later do not allow invalid domains.')
+    @mark.skipif(
+        parse_version(w3lib_version) >= parse_version("2.0.0"),
+        reason="w3lib 2.0.0 and later do not allow invalid domains.",
+    )
     def test_ipv6_default_name_resolver(self):
-        log = self.run_script('default_name_resolver.py')
-        self.assertIn('Spider closed (finished)', log)
-        self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+        log = self.run_script("default_name_resolver.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn(
+            "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,",
+            log,
+        )
         self.assertIn(
             "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
-            log)
+            log,
+        )
 
     def test_caching_hostname_resolver_ipv6(self):
         log = self.run_script("caching_hostname_resolver_ipv6.py")
@@ -406,9 +421,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_twisted_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.selectreactor.SelectReactor", log
+        )
 
-    @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
+    @mark.skipif(
+        platform.system() == "Windows", reason="PollReactor is not supported on Windows"
+    )
     def test_twisted_reactor_poll(self):
         log = self.run_script("twisted_reactor_poll.py")
         self.assertIn("Spider closed (finished)", log)
@@ -417,54 +436,103 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_twisted_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_twisted_reactor_asyncio_custom_settings(self):
         log = self.run_script("twisted_reactor_custom_settings.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_twisted_reactor_asyncio_custom_settings_same(self):
         log = self.run_script("twisted_reactor_custom_settings_same.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
 
     def test_twisted_reactor_asyncio_custom_settings_conflict(self):
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
-        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
-        self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.selectreactor.SelectReactor", log
+        )
+        self.assertIn(
+            "(twisted.internet.selectreactor.SelectReactor) does not match the requested one",
+            log,
+        )
 
-    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
-    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
-    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    @mark.skipif(
+        sys.implementation.name == "pypy",
+        reason="uvloop does not support pypy properly",
+    )
+    @mark.skipif(
+        platform.system() == "Windows", reason="uvloop does not support Windows"
+    )
+    @mark.skipif(
+        twisted_version == Version("twisted", 21, 2, 0),
+        reason="https://twistedmatrix.com/trac/ticket/10106",
+    )
     def test_custom_loop_asyncio(self):
         log = self.run_script("asyncio_custom_loop.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
-    @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
-    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    @mark.skipif(
+        sys.implementation.name == "pypy",
+        reason="uvloop does not support pypy properly",
+    )
+    @mark.skipif(
+        platform.system() == "Windows", reason="uvloop does not support Windows"
+    )
+    @mark.skipif(
+        twisted_version == Version("twisted", 21, 2, 0),
+        reason="https://twistedmatrix.com/trac/ticket/10106",
+    )
     def test_custom_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
-    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
-    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
-    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    @mark.skipif(
+        sys.implementation.name == "pypy",
+        reason="uvloop does not support pypy properly",
+    )
+    @mark.skipif(
+        platform.system() == "Windows", reason="uvloop does not support Windows"
+    )
+    @mark.skipif(
+        twisted_version == Version("twisted", 21, 2, 0),
+        reason="https://twistedmatrix.com/trac/ticket/10106",
+    )
     def test_asyncio_enabled_reactor_same_loop(self):
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
-    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
-    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    @mark.skipif(
+        sys.implementation.name == "pypy",
+        reason="uvloop does not support pypy properly",
+    )
+    @mark.skipif(
+        platform.system() == "Windows", reason="uvloop does not support Windows"
+    )
+    @mark.skipif(
+        twisted_version == Version("twisted", 21, 2, 0),
+        reason="https://twistedmatrix.com/trac/ticket/10106",
+    )
     def test_asyncio_enabled_reactor_different_loop(self):
         log = self.run_script("asyncio_enabled_reactor_different_loop.py")
         self.assertNotIn("Spider closed (finished)", log)
@@ -479,13 +547,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_default_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py")
         self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn(
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
+        )
         self.assertNotIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
 
 class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
-    script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner'
+    script_dir = Path(__file__).parent.resolve() / "CrawlerRunner"
 
     def test_response_ip_address(self):
         log = self.run_script("ip_address.py")
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 74fdd966b..f9b2d853c 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -9,15 +9,14 @@ from twisted.trial import unittest
 
 
 class ScrapyUtilsTest(unittest.TestCase):
-
     def test_required_openssl_version(self):
         try:
-            module = import_module('OpenSSL')
+            module = import_module("OpenSSL")
         except ImportError:
             raise unittest.SkipTest("OpenSSL is not available")
 
-        if hasattr(module, '__version__'):
-            installed_version = [int(x) for x in module.__version__.split('.')[:2]]
+        if hasattr(module, "__version__"):
+            installed_version = [int(x) for x in module.__version__.split(".")[:2]]
             assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
 
     def test_pinned_twisted_version(self):
@@ -27,20 +26,17 @@ class ScrapyUtilsTest(unittest.TestCase):
 
         See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
         """
-        if not os.environ.get('_SCRAPY_PINNED', None):
-            self.skipTest('Not in a pinned environment')
+        if not os.environ.get("_SCRAPY_PINNED", None):
+            self.skipTest("Not in a pinned environment")
 
-        tox_config_file_path = Path(__file__).parent / '..' / 'tox.ini'
+        tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"
         config_parser = ConfigParser()
         config_parser.read(tox_config_file_path)
-        pattern = r'Twisted\[http2\]==([\d.]+)'
-        match = re.search(pattern, config_parser['pinned']['deps'])
+        pattern = r"Twisted\[http2\]==([\d.]+)"
+        match = re.search(pattern, config_parser["pinned"]["deps"])
         pinned_twisted_version_string = match[1]
 
-        self.assertEqual(
-            twisted_version.short(),
-            pinned_twisted_version_string
-        )
+        self.assertEqual(twisted_version.short(), pinned_twisted_version_string)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 3dc2745a0..536d09f18 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -66,50 +66,48 @@ class OffDH:
 
 
 class LoadTestCase(unittest.TestCase):
-
     def test_enabled_handler(self):
-        handlers = {'scheme': DummyDH}
-        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        handlers = {"scheme": DummyDH}
+        crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})
         dh = DownloadHandlers(crawler)
-        self.assertIn('scheme', dh._schemes)
-        self.assertIn('scheme', dh._handlers)
-        self.assertNotIn('scheme', dh._notconfigured)
+        self.assertIn("scheme", dh._schemes)
+        self.assertIn("scheme", dh._handlers)
+        self.assertNotIn("scheme", dh._notconfigured)
 
     def test_not_configured_handler(self):
-        handlers = {'scheme': OffDH}
-        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        handlers = {"scheme": OffDH}
+        crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})
         dh = DownloadHandlers(crawler)
-        self.assertIn('scheme', dh._schemes)
-        self.assertNotIn('scheme', dh._handlers)
-        self.assertIn('scheme', dh._notconfigured)
+        self.assertIn("scheme", dh._schemes)
+        self.assertNotIn("scheme", dh._handlers)
+        self.assertIn("scheme", dh._notconfigured)
 
     def test_disabled_handler(self):
-        handlers = {'scheme': None}
-        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        handlers = {"scheme": None}
+        crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})
         dh = DownloadHandlers(crawler)
-        self.assertNotIn('scheme', dh._schemes)
+        self.assertNotIn("scheme", dh._schemes)
         for scheme in handlers:  # force load handlers
             dh._get_handler(scheme)
-        self.assertNotIn('scheme', dh._handlers)
-        self.assertIn('scheme', dh._notconfigured)
+        self.assertNotIn("scheme", dh._handlers)
+        self.assertIn("scheme", dh._notconfigured)
 
     def test_lazy_handlers(self):
-        handlers = {'scheme': DummyLazyDH}
-        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        handlers = {"scheme": DummyLazyDH}
+        crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})
         dh = DownloadHandlers(crawler)
-        self.assertIn('scheme', dh._schemes)
-        self.assertNotIn('scheme', dh._handlers)
+        self.assertIn("scheme", dh._schemes)
+        self.assertNotIn("scheme", dh._handlers)
         for scheme in handlers:  # force load lazy handler
             dh._get_handler(scheme)
-        self.assertIn('scheme', dh._handlers)
-        self.assertNotIn('scheme', dh._notconfigured)
+        self.assertIn("scheme", dh._handlers)
+        self.assertNotIn("scheme", dh._notconfigured)
 
 
 class FileTestCase(unittest.TestCase):
-
     def setUp(self):
         # add a special char to check that they are handled correctly
-        self.tmpname = Path(self.mktemp() + '^')
+        self.tmpname = Path(self.mktemp() + "^")
         Path(self.tmpname).write_text("0123456789", encoding="utf-8")
         handler = create_instance(FileDownloadHandler, None, get_crawler())
         self.download_request = handler.download_request
@@ -121,16 +119,16 @@ class FileTestCase(unittest.TestCase):
         def _test(response):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.status, 200)
-            self.assertEqual(response.body, b'0123456789')
+            self.assertEqual(response.body, b"0123456789")
             self.assertEqual(response.protocol, None)
 
         request = Request(path_to_file_uri(str(self.tmpname)))
-        assert request.url.upper().endswith('%5E')
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        assert request.url.upper().endswith("%5E")
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_non_existent(self):
-        request = Request(f'file://{self.mktemp()}')
-        d = self.download_request(request, Spider('foo'))
+        request = Request(f"file://{self.mktemp()}")
+        d = self.download_request(request, Spider("foo"))
         return self.assertFailure(d, IOError)
 
 
@@ -145,7 +143,6 @@ class ContentLengthHeaderResource(resource.Resource):
 
 
 class ChunkedResource(resource.Resource):
-
     def render(self, request):
         def response():
             request.write(b"chunked ")
@@ -157,7 +154,6 @@ class ChunkedResource(resource.Resource):
 
 
 class BrokenChunkedResource(resource.Resource):
-
     def render(self, request):
         def response():
             request.write(b"chunked ")
@@ -171,7 +167,6 @@ class BrokenChunkedResource(resource.Resource):
 
 
 class BrokenDownloadResource(resource.Resource):
-
     def render(self, request):
         def response():
             request.setHeader(b"Content-Length", b"20")
@@ -185,7 +180,7 @@ class BrokenDownloadResource(resource.Resource):
 def closeConnection(request):
     # We have to force a disconnection for HTTP/1.1 clients. Otherwise
     # client keeps the connection open waiting for more data.
-    if hasattr(request.channel, 'loseConnection'):  # twisted >=16.3.0
+    if hasattr(request.channel, "loseConnection"):  # twisted >=16.3.0
         request.channel.loseConnection()
     else:
         request.channel.transport.loseConnection()
@@ -215,12 +210,12 @@ class LargeChunkedFileResource(resource.Resource):
 
 
 class HttpTestCase(unittest.TestCase):
-    scheme = 'http'
+    scheme = "http"
     download_handler_cls: Type = HTTPDownloadHandler
 
     # only used for HTTPS tests
-    keyfile = 'keys/localhost.key'
-    certfile = 'keys/localhost.crt'
+    keyfile = "keys/localhost.key"
+    certfile = "keys/localhost.crt"
 
     def setUp(self):
         self.tmpname = Path(self.mktemp())
@@ -242,23 +237,28 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
-        self.host = 'localhost'
-        if self.scheme == 'https':
+        self.host = "localhost"
+        if self.scheme == "https":
             # Using WrappingFactory do not enable HTTP/2 failing all the
             # tests with H2DownloadHandler
             self.port = reactor.listenSSL(
-                0, self.site, ssl_context_factory(self.keyfile, self.certfile),
-                interface=self.host)
+                0,
+                self.site,
+                ssl_context_factory(self.keyfile, self.certfile),
+                interface=self.host,
+            )
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
-        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
+        self.download_handler = create_instance(
+            self.download_handler_cls, None, get_crawler()
+        )
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
     def tearDown(self):
         yield self.port.stopListening()
-        if hasattr(self.download_handler, 'close'):
+        if hasattr(self.download_handler, "close"):
             yield self.download_handler.close()
         shutil.rmtree(self.tmpname)
 
@@ -266,29 +266,29 @@ class HttpTestCase(unittest.TestCase):
         return f"{self.scheme}://{self.host}:{self.portno}/{path}"
 
     def test_download(self):
-        request = Request(self.getURL('file'))
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("file"))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_download_head(self):
-        request = Request(self.getURL('file'), method='HEAD')
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("file"), method="HEAD")
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b'')
+        d.addCallback(self.assertEqual, b"")
         return d
 
     def test_redirect_status(self):
-        request = Request(self.getURL('redirect'))
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("redirect"))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.status)
         d.addCallback(self.assertEqual, 302)
         return d
 
     def test_redirect_status_head(self):
-        request = Request(self.getURL('redirect'), method='HEAD')
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("redirect"), method="HEAD")
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.status)
         d.addCallback(self.assertEqual, 302)
         return d
@@ -302,9 +302,9 @@ class HttpTestCase(unittest.TestCase):
             )
 
         # client connects but no data is received
-        spider = Spider('foo')
-        meta = {'download_timeout': 0.5}
-        request = Request(self.getURL('wait'), meta=meta)
+        spider = Spider("foo")
+        meta = {"download_timeout": 0.5}
+        request = Request(self.getURL("wait"), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
 
@@ -316,34 +316,33 @@ class HttpTestCase(unittest.TestCase):
                 "This test produces DirtyReactorAggregateError on Windows with asyncio"
             )
         # client connects, server send headers and some body bytes but hangs
-        spider = Spider('foo')
-        meta = {'download_timeout': 0.5}
-        request = Request(self.getURL('hang-after-headers'), meta=meta)
+        spider = Spider("foo")
+        meta = {"download_timeout": 0.5}
+        request = Request(self.getURL("hang-after-headers"), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
 
     def test_host_header_not_in_request_headers(self):
         def _test(response):
-            self.assertEqual(
-                response.body, to_bytes(f'{self.host}:{self.portno}'))
+            self.assertEqual(response.body, to_bytes(f"{self.host}:{self.portno}"))
             self.assertEqual(request.headers, {})
 
-        request = Request(self.getURL('host'))
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        request = Request(self.getURL("host"))
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_host_header_seted_in_request_headers(self):
-        host = self.host + ':' + str(self.portno)
+        host = self.host + ":" + str(self.portno)
 
         def _test(response):
             self.assertEqual(response.body, host.encode())
-            self.assertEqual(request.headers.get('Host'), host.encode())
+            self.assertEqual(request.headers.get("Host"), host.encode())
 
-        request = Request(self.getURL('host'), headers={'Host': host})
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        request = Request(self.getURL("host"), headers={"Host": host})
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
-        d = self.download_request(request, Spider('foo'))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b'localhost')
+        d.addCallback(self.assertEqual, b"localhost")
         return d
 
     def test_content_length_zero_bodyless_post_request_headers(self):
@@ -359,26 +358,27 @@ class HttpTestCase(unittest.TestCase):
         """
 
         def _test(response):
-            self.assertEqual(response.body, b'0')
+            self.assertEqual(response.body, b"0")
 
-        request = Request(self.getURL('contentlength'), method='POST')
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        request = Request(self.getURL("contentlength"), method="POST")
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_content_length_zero_bodyless_post_only_one(self):
         def _test(response):
             import json
-            headers = Headers(json.loads(response.text)['headers'])
-            contentlengths = headers.getlist('Content-Length')
+
+            headers = Headers(json.loads(response.text)["headers"])
+            contentlengths = headers.getlist("Content-Length")
             self.assertEqual(len(contentlengths), 1)
             self.assertEqual(contentlengths, [b"0"])
 
-        request = Request(self.getURL('echo'), method='POST')
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        request = Request(self.getURL("echo"), method="POST")
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_payload(self):
-        body = b'1' * 100  # PayloadResource requires body length to be 100
-        request = Request(self.getURL('payload'), method='POST', body=body)
-        d = self.download_request(request, Spider('foo'))
+        body = b"1" * 100  # PayloadResource requires body length to be 100
+        request = Request(self.getURL("payload"), method="POST", body=body)
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, body)
         return d
@@ -386,8 +386,8 @@ class HttpTestCase(unittest.TestCase):
     def test_response_header_content_length(self):
         request = Request(self.getURL("file"), method=b"GET")
         d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.headers[b'content-length'])
-        d.addCallback(self.assertEqual, b'159')
+        d.addCallback(lambda r: r.headers[b"content-length"])
+        d.addCallback(self.assertEqual, b"159")
         return d
 
     def _test_response_class(self, filename, body, response_class):
@@ -395,14 +395,14 @@ class HttpTestCase(unittest.TestCase):
             self.assertEqual(type(response), response_class)
 
         request = Request(self.getURL(filename), body=body)
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_response_class_from_url(self):
-        return self._test_response_class('foo.html', b'', HtmlResponse)
+        return self._test_response_class("foo.html", b"", HtmlResponse)
 
     def test_response_class_from_body(self):
         return self._test_response_class(
-            'foo',
+            "foo",
             b"<!DOCTYPE html>\n<title>.</title>",
             HtmlResponse,
         )
@@ -410,6 +410,7 @@ class HttpTestCase(unittest.TestCase):
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
+
     download_handler_cls: Type = HTTP10DownloadHandler
 
     def test_protocol(self):
@@ -421,57 +422,58 @@ class Http10TestCase(HttpTestCase):
 
 
 class Https10TestCase(Http10TestCase):
-    scheme = 'https'
+    scheme = "https"
 
 
 class Http11TestCase(HttpTestCase):
     """HTTP 1.1 test case"""
+
     download_handler_cls: Type = HTTP11DownloadHandler
 
     def test_download_without_maxsize_limit(self):
-        request = Request(self.getURL('file'))
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("file"))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_response_class_choosing_request(self):
         """Tests choosing of correct response type
-         in case of Content-Type is empty but body contains text.
+        in case of Content-Type is empty but body contains text.
         """
-        body = b'Some plain text\ndata with tabs\t and null bytes\0'
+        body = b"Some plain text\ndata with tabs\t and null bytes\0"
 
         def _test_type(response):
             self.assertEqual(type(response), TextResponse)
 
-        request = Request(self.getURL('nocontenttype'), body=body)
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("nocontenttype"), body=body)
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(_test_type)
         return d
 
     @defer.inlineCallbacks
     def test_download_with_maxsize(self):
-        request = Request(self.getURL('file'))
+        request = Request(self.getURL("file"))
 
         # 10 is minimal size for this request and the limit is only counted on
         # response body. (regardless of headers)
-        d = self.download_request(request, Spider('foo', download_maxsize=10))
+        d = self.download_request(request, Spider("foo", download_maxsize=10))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, b"0123456789")
         yield d
 
-        d = self.download_request(request, Spider('foo', download_maxsize=9))
+        d = self.download_request(request, Spider("foo", download_maxsize=9))
         yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
     @defer.inlineCallbacks
     def test_download_with_maxsize_very_large_file(self):
-        with mock.patch('scrapy.core.downloader.handlers.http11.logger') as logger:
-            request = Request(self.getURL('largechunkedfile'))
+        with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:
+            request = Request(self.getURL("largechunkedfile"))
 
             def check(logger):
                 logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
 
-            d = self.download_request(request, Spider('foo', download_maxsize=1500))
+            d = self.download_request(request, Spider("foo", download_maxsize=1500))
             yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
             # As the error message is logged in the dataReceived callback, we
@@ -479,39 +481,39 @@ class Http11TestCase(HttpTestCase):
             # after closing the connection.
             d = defer.Deferred()
             d.addCallback(check)
-            reactor.callLater(.1, d.callback, logger)
+            reactor.callLater(0.1, d.callback, logger)
             yield d
 
     @defer.inlineCallbacks
     def test_download_with_maxsize_per_req(self):
-        meta = {'download_maxsize': 2}
-        request = Request(self.getURL('file'), meta=meta)
-        d = self.download_request(request, Spider('foo'))
+        meta = {"download_maxsize": 2}
+        request = Request(self.getURL("file"), meta=meta)
+        d = self.download_request(request, Spider("foo"))
         yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
     @defer.inlineCallbacks
     def test_download_with_small_maxsize_per_spider(self):
-        request = Request(self.getURL('file'))
-        d = self.download_request(request, Spider('foo', download_maxsize=2))
+        request = Request(self.getURL("file"))
+        d = self.download_request(request, Spider("foo", download_maxsize=2))
         yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
     def test_download_with_large_maxsize_per_spider(self):
-        request = Request(self.getURL('file'))
-        d = self.download_request(request, Spider('foo', download_maxsize=100))
+        request = Request(self.getURL("file"))
+        d = self.download_request(request, Spider("foo", download_maxsize=100))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_download_chunked_content(self):
-        request = Request(self.getURL('chunked'))
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("chunked"))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, b"chunked content\n")
         return d
 
-    def test_download_broken_content_cause_data_loss(self, url='broken'):
+    def test_download_broken_content_cause_data_loss(self, url="broken"):
         request = Request(self.getURL(url))
-        d = self.download_request(request, Spider('foo'))
+        d = self.download_request(request, Spider("foo"))
 
         def checkDataLoss(failure):
             if failure.check(ResponseFailed):
@@ -524,29 +526,31 @@ class Http11TestCase(HttpTestCase):
         return d
 
     def test_download_broken_chunked_content_cause_data_loss(self):
-        return self.test_download_broken_content_cause_data_loss('broken-chunked')
+        return self.test_download_broken_content_cause_data_loss("broken-chunked")
 
-    def test_download_broken_content_allow_data_loss(self, url='broken'):
-        request = Request(self.getURL(url), meta={'download_fail_on_dataloss': False})
-        d = self.download_request(request, Spider('foo'))
+    def test_download_broken_content_allow_data_loss(self, url="broken"):
+        request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False})
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.flags)
-        d.addCallback(self.assertEqual, ['dataloss'])
+        d.addCallback(self.assertEqual, ["dataloss"])
         return d
 
     def test_download_broken_chunked_content_allow_data_loss(self):
-        return self.test_download_broken_content_allow_data_loss('broken-chunked')
+        return self.test_download_broken_content_allow_data_loss("broken-chunked")
 
-    def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
-        crawler = get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
+    def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"):
+        crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False})
         download_handler = create_instance(self.download_handler_cls, None, crawler)
         request = Request(self.getURL(url))
-        d = download_handler.download_request(request, Spider('foo'))
+        d = download_handler.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.flags)
-        d.addCallback(self.assertEqual, ['dataloss'])
+        d.addCallback(self.assertEqual, ["dataloss"])
         return d
 
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
-        return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked')
+        return self.test_download_broken_content_allow_data_loss_via_setting(
+            "broken-chunked"
+        )
 
     def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
@@ -557,7 +561,7 @@ class Http11TestCase(HttpTestCase):
 
 
 class Https11TestCase(Http11TestCase):
-    scheme = 'https'
+    scheme = "https"
 
     tls_log_message = (
         'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
@@ -566,30 +570,34 @@ class Https11TestCase(Http11TestCase):
 
     @defer.inlineCallbacks
     def test_tls_logging(self):
-        crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
+        crawler = get_crawler(
+            settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}
+        )
         download_handler = create_instance(self.download_handler_cls, None, crawler)
         try:
             with LogCapture() as log_capture:
-                request = Request(self.getURL('file'))
-                d = download_handler.download_request(request, Spider('foo'))
+                request = Request(self.getURL("file"))
+                d = download_handler.download_request(request, Spider("foo"))
                 d.addCallback(lambda r: r.body)
                 d.addCallback(self.assertEqual, b"0123456789")
                 yield d
-                log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message))
+                log_capture.check_present(
+                    ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message)
+                )
         finally:
             yield download_handler.close()
 
 
 class Https11WrongHostnameTestCase(Http11TestCase):
-    scheme = 'https'
+    scheme = "https"
 
     # above tests use a server certificate for "localhost",
     # client connection to "localhost" too.
     # here we test that even if the server certificate is for another domain,
     # "www.example.com" in this case,
     # the tests still pass
-    keyfile = 'keys/example-com.key.pem'
-    certfile = 'keys/example-com.cert.pem'
+    keyfile = "keys/example-com.key.pem"
+    certfile = "keys/example-com.cert.pem"
 
 
 class Https11InvalidDNSId(Https11TestCase):
@@ -597,14 +605,14 @@ class Https11InvalidDNSId(Https11TestCase):
 
     def setUp(self):
         super().setUp()
-        self.host = '127.0.0.1'
+        self.host = "127.0.0.1"
 
 
 class Https11InvalidDNSPattern(Https11TestCase):
     """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
 
-    keyfile = 'keys/localhost.ip.key'
-    certfile = 'keys/localhost.ip.crt'
+    keyfile = "keys/localhost.ip.key"
+    certfile = "keys/localhost.ip.crt"
 
     def setUp(self):
         try:
@@ -619,11 +627,11 @@ class Https11InvalidDNSPattern(Https11TestCase):
 
 
 class Https11CustomCiphers(unittest.TestCase):
-    scheme = 'https'
+    scheme = "https"
     download_handler_cls: Type = HTTP11DownloadHandler
 
-    keyfile = 'keys/localhost.key'
-    certfile = 'keys/localhost.crt'
+    keyfile = "keys/localhost.key"
+    certfile = "keys/localhost.crt"
 
     def setUp(self):
         self.tmpname = Path(self.mktemp())
@@ -631,19 +639,28 @@ class Https11CustomCiphers(unittest.TestCase):
         (self.tmpname / "file").write_bytes(b"0123456789")
         r = static.File(str(self.tmpname))
         self.site = server.Site(r, timeout=None)
-        self.host = 'localhost'
+        self.host = "localhost"
         self.port = reactor.listenSSL(
-            0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
-            interface=self.host)
+            0,
+            self.site,
+            ssl_context_factory(
+                self.keyfile, self.certfile, cipher_string="CAMELLIA256-SHA"
+            ),
+            interface=self.host,
+        )
         self.portno = self.port.getHost().port
-        crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
-        self.download_handler = create_instance(self.download_handler_cls, None, crawler)
+        crawler = get_crawler(
+            settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"}
+        )
+        self.download_handler = create_instance(
+            self.download_handler_cls, None, crawler
+        )
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
     def tearDown(self):
         yield self.port.stopListening()
-        if hasattr(self.download_handler, 'close'):
+        if hasattr(self.download_handler, "close"):
             yield self.download_handler.close()
         shutil.rmtree(self.tmpname)
 
@@ -651,8 +668,8 @@ class Https11CustomCiphers(unittest.TestCase):
         return f"{self.scheme}://{self.host}:{self.portno}/{path}"
 
     def test_download(self):
-        request = Request(self.getURL('file'))
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("file"))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEqual, b"0123456789")
         return d
@@ -660,6 +677,7 @@ class Https11CustomCiphers(unittest.TestCase):
 
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
+
     settings_dict: Optional[dict] = None
 
     def setUp(self):
@@ -674,40 +692,48 @@ class Http11MockServerTestCase(unittest.TestCase):
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
         # download it
-        yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000}))
-        failure = crawler.spider.meta['failure']
+        yield crawler.crawl(
+            seed=Request(
+                url=self.mockserver.url("/partial"), meta={"download_maxsize": 1000}
+            )
+        )
+        failure = crawler.spider.meta["failure"]
         self.assertIsInstance(failure.value, defer.CancelledError)
 
     @defer.inlineCallbacks
     def test_download(self):
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
-        yield crawler.crawl(seed=Request(url=self.mockserver.url('')))
-        failure = crawler.spider.meta.get('failure')
+        yield crawler.crawl(seed=Request(url=self.mockserver.url("")))
+        failure = crawler.spider.meta.get("failure")
         self.assertTrue(failure is None)
-        reason = crawler.spider.meta['close_reason']
-        self.assertTrue(reason, 'finished')
+        reason = crawler.spider.meta["close_reason"]
+        self.assertTrue(reason, "finished")
 
     @defer.inlineCallbacks
     def test_download_gzip_response(self):
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
-        body = b'1' * 100  # PayloadResource requires body length to be 100
-        request = Request(self.mockserver.url('/payload'), method='POST',
-                          body=body, meta={'download_maxsize': 50})
+        body = b"1" * 100  # PayloadResource requires body length to be 100
+        request = Request(
+            self.mockserver.url("/payload"),
+            method="POST",
+            body=body,
+            meta={"download_maxsize": 50},
+        )
         yield crawler.crawl(seed=request)
-        failure = crawler.spider.meta['failure']
+        failure = crawler.spider.meta["failure"]
         # download_maxsize < 100, hence the CancelledError
         self.assertIsInstance(failure.value, defer.CancelledError)
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
         raise unittest.SkipTest("xpayload fails on PY3")
-        request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-        request = request.replace(url=self.mockserver.url('/xpayload'))
+        request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate")
+        request = request.replace(url=self.mockserver.url("/xpayload"))
         yield crawler.crawl(seed=request)
         # download_maxsize = 50 is enough for the gzipped response
-        failure = crawler.spider.meta.get('failure')
+        failure = crawler.spider.meta.get("failure")
         self.assertTrue(failure is None)
-        reason = crawler.spider.meta['close_reason']
-        self.assertTrue(reason, 'finished')
+        reason = crawler.spider.meta["close_reason"]
+        self.assertTrue(reason, "finished")
 
 
 class UriResource(resource.Resource):
@@ -720,27 +746,29 @@ class UriResource(resource.Resource):
         # Note: this is an ugly hack for CONNECT request timeout test.
         #       Returning some data here fail SSL/TLS handshake
         # ToDo: implement proper HTTPS proxy tests, not faking them.
-        if request.method != b'CONNECT':
+        if request.method != b"CONNECT":
             return request.uri
-        return b''
+        return b""
 
 
 class HttpProxyTestCase(unittest.TestCase):
     download_handler_cls: Type = HTTPDownloadHandler
-    expected_http_proxy_request_body = b'http://example.com'
+    expected_http_proxy_request_body = b"http://example.com"
 
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
         wrapper = WrappingFactory(site)
-        self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
+        self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1")
         self.portno = self.port.getHost().port
-        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
+        self.download_handler = create_instance(
+            self.download_handler_cls, None, get_crawler()
+        )
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
     def tearDown(self):
         yield self.port.stopListening()
-        if hasattr(self.download_handler, 'close'):
+        if hasattr(self.download_handler, "close"):
             yield self.download_handler.close()
 
     def getURL(self, path):
@@ -752,25 +780,25 @@ class HttpProxyTestCase(unittest.TestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.body, self.expected_http_proxy_request_body)
 
-        http_proxy = self.getURL('')
-        request = Request('http://example.com', meta={'proxy': http_proxy})
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        http_proxy = self.getURL("")
+        request = Request("http://example.com", meta={"proxy": http_proxy})
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_download_without_proxy(self):
         def _test(response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.url, request.url)
-            self.assertEqual(response.body, b'/path/to/resource')
+            self.assertEqual(response.body, b"/path/to/resource")
 
-        request = Request(self.getURL('path/to/resource'))
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        request = Request(self.getURL("path/to/resource"))
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
 
 class Http10ProxyTestCase(HttpProxyTestCase):
     download_handler_cls: Type = HTTP10DownloadHandler
 
     def test_download_with_proxy_https_noconnect(self):
-        raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler')
+        raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler")
 
 
 class Http11ProxyTestCase(HttpProxyTestCase):
@@ -778,14 +806,13 @@ class Http11ProxyTestCase(HttpProxyTestCase):
 
     @defer.inlineCallbacks
     def test_download_with_proxy_https_timeout(self):
-        """ Test TunnelingTCP4ClientEndpoint """
+        """Test TunnelingTCP4ClientEndpoint"""
         if NON_EXISTING_RESOLVABLE:
             raise SkipTest("Non-existing hosts are resolvable")
-        http_proxy = self.getURL('')
-        domain = 'https://no-such-domain.nosuch'
-        request = Request(
-            domain, meta={'proxy': http_proxy, 'download_timeout': 0.2})
-        d = self.download_request(request, Spider('foo'))
+        http_proxy = self.getURL("")
+        domain = "https://no-such-domain.nosuch"
+        request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})
+        d = self.download_request(request, Spider("foo"))
         timeout = yield self.assertFailure(d, error.TimeoutError)
         self.assertIn(domain, timeout.osError)
 
@@ -795,13 +822,12 @@ class Http11ProxyTestCase(HttpProxyTestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.body, self.expected_http_proxy_request_body)
 
-        http_proxy = self.getURL('').replace('http://', '')
-        request = Request('http://example.com', meta={'proxy': http_proxy})
-        return self.download_request(request, Spider('foo')).addCallback(_test)
+        http_proxy = self.getURL("").replace("http://", "")
+        request = Request("http://example.com", meta={"proxy": http_proxy})
+        return self.download_request(request, Spider("foo")).addCallback(_test)
 
 
 class HttpDownloadHandlerMock:
-
     def __init__(self, *args, **kwargs):
         pass
 
@@ -810,7 +836,6 @@ class HttpDownloadHandlerMock:
 
 
 class S3AnonTestCase(unittest.TestCase):
-
     def setUp(self):
         skip_if_no_boto()
         crawler = get_crawler()
@@ -822,15 +847,14 @@ class S3AnonTestCase(unittest.TestCase):
             # anon=True, # implicit
         )
         self.download_request = self.s3reqh.download_request
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
 
     def test_anon_request(self):
-        req = Request('s3://aws-publicdatasets/')
+        req = Request("s3://aws-publicdatasets/")
         httpreq = self.download_request(req, self.spider)
-        self.assertEqual(hasattr(self.s3reqh, 'anon'), True)
+        self.assertEqual(hasattr(self.s3reqh, "anon"), True)
         self.assertEqual(self.s3reqh.anon, True)
-        self.assertEqual(
-            httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
+        self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/")
 
 
 class S3TestCase(unittest.TestCase):
@@ -840,8 +864,8 @@ class S3TestCase(unittest.TestCase):
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
     # and the tests described here are the examples from that manual
 
-    AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82'
-    AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
+    AWS_ACCESS_KEY_ID = "0PN5J17HBGZHT7JJ3X82"
+    AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"
 
     def setUp(self):
         skip_if_no_boto()
@@ -855,7 +879,7 @@ class S3TestCase(unittest.TestCase):
             httpdownloadhandler=HttpDownloadHandlerMock,
         )
         self.download_request = s3reqh.download_request
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
 
     @contextlib.contextmanager
     def _mocked_date(self, date):
@@ -867,7 +891,7 @@ class S3TestCase(unittest.TestCase):
             # We need to mock botocore.auth.formatdate, because otherwise
             # botocore overrides Date header with current date and time
             # and Authorization header is different each time
-            with mock.patch('botocore.auth.formatdate') as mock_formatdate:
+            with mock.patch("botocore.auth.formatdate") as mock_formatdate:
                 mock_formatdate.return_value = date
                 yield
 
@@ -887,90 +911,105 @@ class S3TestCase(unittest.TestCase):
 
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
-        date = 'Tue, 27 Mar 2007 19:36:42 +0000'
-        req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
+        date = "Tue, 27 Mar 2007 19:36:42 +0000"
+        req = Request("s3://johnsmith/photos/puppy.jpg", headers={"Date": date})
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'],
-                         b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
+        self.assertEqual(
+            httpreq.headers["Authorization"],
+            b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=",
+        )
 
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
-        date = 'Tue, 27 Mar 2007 21:15:45 +0000'
+        date = "Tue, 27 Mar 2007 21:15:45 +0000"
         req = Request(
-            's3://johnsmith/photos/puppy.jpg',
-            method='PUT',
+            "s3://johnsmith/photos/puppy.jpg",
+            method="PUT",
             headers={
-                'Content-Type': 'image/jpeg',
-                'Date': date,
-                'Content-Length': '94328',
+                "Content-Type": "image/jpeg",
+                "Date": date,
+                "Content-Length": "94328",
             },
         )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'],
-                         b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
+        self.assertEqual(
+            httpreq.headers["Authorization"],
+            b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=",
+        )
 
     def test_request_signing3(self):
         # lists the content of the johnsmith bucket.
-        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
+        date = "Tue, 27 Mar 2007 19:42:41 +0000"
         req = Request(
-            's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy',
-            method='GET', headers={
-                'User-Agent': 'Mozilla/5.0',
-                'Date': date,
-            })
-        with self._mocked_date(date):
-            httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'],
-                         b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
-
-    def test_request_signing4(self):
-        # fetches the access control policy sub-resource for the 'johnsmith' bucket.
-        date = 'Tue, 27 Mar 2007 19:44:46 +0000'
-        req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date})
-        with self._mocked_date(date):
-            httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'],
-                         b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
-
-    def test_request_signing6(self):
-        # uploads an object to a CNAME style virtual hosted bucket with metadata.
-        date = 'Tue, 27 Mar 2007 21:06:08 +0000'
-        req = Request(
-            's3://static.johnsmith.net:8080/db-backup.dat.gz',
-            method='PUT', headers={
-                'User-Agent': 'curl/7.15.5',
-                'Host': 'static.johnsmith.net:8080',
-                'Date': date,
-                'x-amz-acl': 'public-read',
-                'content-type': 'application/x-download',
-                'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
-                'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
-                'X-Amz-Meta-FileChecksum': '0x02661779',
-                'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
-                'Content-Disposition': 'attachment; filename=database.dat',
-                'Content-Encoding': 'gzip',
-                'Content-Length': '5913339',
-            })
-        with self._mocked_date(date):
-            httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'],
-                         b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
-
-    def test_request_signing7(self):
-        # ensure that spaces are quoted properly before signing
-        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
-        req = Request(
-            "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
-            method='GET',
-            headers={'Date': date},
+            "s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy",
+            method="GET",
+            headers={
+                "User-Agent": "Mozilla/5.0",
+                "Date": date,
+            },
         )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(
-            httpreq.headers['Authorization'],
-            b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
+            httpreq.headers["Authorization"],
+            b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=",
+        )
+
+    def test_request_signing4(self):
+        # fetches the access control policy sub-resource for the 'johnsmith' bucket.
+        date = "Tue, 27 Mar 2007 19:44:46 +0000"
+        req = Request("s3://johnsmith/?acl", method="GET", headers={"Date": date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
+        self.assertEqual(
+            httpreq.headers["Authorization"],
+            b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=",
+        )
+
+    def test_request_signing6(self):
+        # uploads an object to a CNAME style virtual hosted bucket with metadata.
+        date = "Tue, 27 Mar 2007 21:06:08 +0000"
+        req = Request(
+            "s3://static.johnsmith.net:8080/db-backup.dat.gz",
+            method="PUT",
+            headers={
+                "User-Agent": "curl/7.15.5",
+                "Host": "static.johnsmith.net:8080",
+                "Date": date,
+                "x-amz-acl": "public-read",
+                "content-type": "application/x-download",
+                "Content-MD5": "4gJE4saaMU4BqNR0kLY+lw==",
+                "X-Amz-Meta-ReviewedBy": "joe@johnsmith.net,jane@johnsmith.net",
+                "X-Amz-Meta-FileChecksum": "0x02661779",
+                "X-Amz-Meta-ChecksumAlgorithm": "crc32",
+                "Content-Disposition": "attachment; filename=database.dat",
+                "Content-Encoding": "gzip",
+                "Content-Length": "5913339",
+            },
+        )
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
+        self.assertEqual(
+            httpreq.headers["Authorization"],
+            b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=",
+        )
+
+    def test_request_signing7(self):
+        # ensure that spaces are quoted properly before signing
+        date = "Tue, 27 Mar 2007 19:42:41 +0000"
+        req = Request(
+            "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
+            method="GET",
+            headers={"Date": date},
+        )
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
+        self.assertEqual(
+            httpreq.headers["Authorization"],
+            b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=",
+        )
 
 
 class BaseFTPTestCase(unittest.TestCase):
@@ -979,9 +1018,9 @@ class BaseFTPTestCase(unittest.TestCase):
     req_meta = {"ftp_user": username, "ftp_password": password}
 
     test_files = (
-        ('file.txt', b"I have the power!"),
-        ('file with spaces.txt', b"Moooooooooo power!"),
-        ('html-file-without-extension', b"<!DOCTYPE html>\n<title>.</title>"),
+        ("file.txt", b"I have the power!"),
+        ("file with spaces.txt", b"Moooooooooo power!"),
+        ("html-file-without-extension", b"<!DOCTYPE html>\n<title>.</title>"),
     )
 
     def setUp(self):
@@ -997,7 +1036,9 @@ class BaseFTPTestCase(unittest.TestCase):
             (userdir / filename).write_bytes(content)
 
         # setup server
-        realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory))
+        realm = FTPRealm(
+            anonymousRoot=str(self.directory), userHome=str(self.directory)
+        )
         p = portal.Portal(realm)
         users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
         users_checker.addUser(self.username, self.password)
@@ -1006,7 +1047,9 @@ class BaseFTPTestCase(unittest.TestCase):
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
         crawler = get_crawler()
-        self.download_handler = create_instance(FTPDownloadHandler, crawler.settings, crawler)
+        self.download_handler = create_instance(
+            FTPDownloadHandler, crawler.settings, crawler
+        )
         self.addCleanup(self.port.stopListening)
 
     def tearDown(self):
@@ -1025,34 +1068,37 @@ class BaseFTPTestCase(unittest.TestCase):
         return deferred
 
     def test_ftp_download_success(self):
-        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",
-                          meta=self.req_meta)
-        d = self.download_handler.download_request(request, None)
-
-        def _test(r):
-            self.assertEqual(r.status, 200)
-            self.assertEqual(r.body, b'I have the power!')
-            self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
-            self.assertIsNone(r.protocol)
-        return self._add_test_callbacks(d, _test)
-
-    def test_ftp_download_path_with_spaces(self):
         request = Request(
-            url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt",
-            meta=self.req_meta
+            url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta
         )
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
             self.assertEqual(r.status, 200)
-            self.assertEqual(r.body, b'Moooooooooo power!')
-            self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']})
+            self.assertEqual(r.body, b"I have the power!")
+            self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"17"]})
+            self.assertIsNone(r.protocol)
+
+        return self._add_test_callbacks(d, _test)
+
+    def test_ftp_download_path_with_spaces(self):
+        request = Request(
+            url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt",
+            meta=self.req_meta,
+        )
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(r.status, 200)
+            self.assertEqual(r.body, b"Moooooooooo power!")
+            self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"18"]})
 
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_notexist(self):
-        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt",
-                          meta=self.req_meta)
+        request = Request(
+            url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta
+        )
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -1067,14 +1113,14 @@ class BaseFTPTestCase(unittest.TestCase):
         os.close(f)
         meta = {"ftp_local_filename": fname_bytes}
         meta.update(self.req_meta)
-        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",
-                          meta=meta)
+        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
             self.assertEqual(r.body, fname_bytes)
-            self.assertEqual(r.headers, {b'Local Filename': [fname_bytes],
-                                         b'Size': [b'17']})
+            self.assertEqual(
+                r.headers, {b"Local Filename": [fname_bytes], b"Size": [b"17"]}
+            )
             self.assertTrue(local_fname.exists())
             self.assertEqual(local_fname.read_bytes(), b"I have the power!")
             local_fname.unlink()
@@ -1087,24 +1133,23 @@ class BaseFTPTestCase(unittest.TestCase):
         os.close(f)
         meta = {}
         meta.update(self.req_meta)
-        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}",
-                          meta=meta)
+        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}", meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
             self.assertEqual(type(r), response_class)
             local_fname.unlink()
+
         return self._add_test_callbacks(d, _test)
 
     def test_response_class_from_url(self):
-        return self._test_response_class('file.txt', TextResponse)
+        return self._test_response_class("file.txt", TextResponse)
 
     def test_response_class_from_body(self):
-        return self._test_response_class('html-file-without-extension', HtmlResponse)
+        return self._test_response_class("html-file-without-extension", HtmlResponse)
 
 
 class FTPTestCase(BaseFTPTestCase):
-
     def test_invalid_credentials(self):
         if self.reactor_pytest == "asyncio" and sys.platform == "win32":
             raise unittest.SkipTest(
@@ -1113,9 +1158,8 @@ class FTPTestCase(BaseFTPTestCase):
         from twisted.protocols.ftp import ConnectionLost
 
         meta = dict(self.req_meta)
-        meta.update({"ftp_password": 'invalid'})
-        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",
-                          meta=meta)
+        meta.update({"ftp_password": "invalid"})
+        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -1142,15 +1186,15 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
         # setup server for anonymous access
         realm = FTPRealm(anonymousRoot=str(self.directory))
         p = portal.Portal(realm)
-        p.registerChecker(checkers.AllowAnonymousAccess(),
-                          credentials.IAnonymous)
+        p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
 
-        self.factory = FTPFactory(portal=p,
-                                  userAnonymous=self.username)
+        self.factory = FTPFactory(portal=p, userAnonymous=self.username)
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
         crawler = get_crawler()
-        self.download_handler = create_instance(FTPDownloadHandler, crawler.settings, crawler)
+        self.download_handler = create_instance(
+            FTPDownloadHandler, crawler.settings, crawler
+        )
         self.addCleanup(self.port.stopListening)
 
     def tearDown(self):
@@ -1158,12 +1202,13 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
 
 
 class DataURITestCase(unittest.TestCase):
-
     def setUp(self):
         crawler = get_crawler()
-        self.download_handler = create_instance(DataURIDownloadHandler, crawler.settings, crawler)
+        self.download_handler = create_instance(
+            DataURIDownloadHandler, crawler.settings, crawler
+        )
         self.download_request = self.download_handler.download_request
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
 
     def test_response_attrs(self):
         uri = "data:,A%20brief%20note"
@@ -1177,7 +1222,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_default_mediatype_encoding(self):
         def _test(response):
-            self.assertEqual(response.text, 'A brief note')
+            self.assertEqual(response.text, "A brief note")
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "US-ASCII")
 
@@ -1186,7 +1231,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_default_mediatype(self):
         def _test(response):
-            self.assertEqual(response.text, '\u038e\u03a3\u038e')
+            self.assertEqual(response.text, "\u038e\u03a3\u038e")
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1195,8 +1240,8 @@ class DataURITestCase(unittest.TestCase):
 
     def test_text_charset(self):
         def _test(response):
-            self.assertEqual(response.text, '\u038e\u03a3\u038e')
-            self.assertEqual(response.body, b'\xbe\xd3\xbe')
+            self.assertEqual(response.text, "\u038e\u03a3\u038e")
+            self.assertEqual(response.body, b"\xbe\xd3\xbe")
             self.assertEqual(response.encoding, "iso-8859-7")
 
         request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
@@ -1204,20 +1249,22 @@ class DataURITestCase(unittest.TestCase):
 
     def test_mediatype_parameters(self):
         def _test(response):
-            self.assertEqual(response.text, '\u038e\u03a3\u038e')
+            self.assertEqual(response.text, "\u038e\u03a3\u038e")
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
-        request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
-                          'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22'
-                          ',%CE%8E%CE%A3%CE%8E')
+        request = Request(
+            "data:text/plain;foo=%22foo;bar%5C%22%22;"
+            "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22"
+            ",%CE%8E%CE%A3%CE%8E"
+        )
         return self.download_request(request, self.spider).addCallback(_test)
 
     def test_base64(self):
         def _test(response):
-            self.assertEqual(response.text, 'Hello, world.')
+            self.assertEqual(response.text, "Hello, world.")
 
-        request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
+        request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D")
         return self.download_request(request, self.spider).addCallback(_test)
 
     def test_protocol(self):
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index c2fa3ec57..fd765089a 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -15,21 +15,24 @@ from scrapy.utils.misc import create_instance
 from scrapy.utils.test import get_crawler
 from tests.mockserver import ssl_context_factory
 from tests.test_downloader_handlers import (
-    Https11TestCase, Https11CustomCiphers,
-    Http11MockServerTestCase, Http11ProxyTestCase,
-    UriResource
+    Https11TestCase,
+    Https11CustomCiphers,
+    Http11MockServerTestCase,
+    Http11ProxyTestCase,
+    UriResource,
 )
 
 
 @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2TestCase(Https11TestCase):
 
-    scheme = 'https'
+    scheme = "https"
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
     @classmethod
     def setUpClass(cls):
         from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+
         cls.download_handler_cls = H2DownloadHandler
 
     def test_protocol(self):
@@ -41,13 +44,13 @@ class Https2TestCase(Https11TestCase):
 
     @defer.inlineCallbacks
     def test_download_with_maxsize_very_large_file(self):
-        with mock.patch('scrapy.core.http2.stream.logger') as logger:
-            request = Request(self.getURL('largechunkedfile'))
+        with mock.patch("scrapy.core.http2.stream.logger") as logger:
+            request = Request(self.getURL("largechunkedfile"))
 
             def check(logger):
                 logger.error.assert_called_once_with(mock.ANY)
 
-            d = self.download_request(request, Spider('foo', download_maxsize=1500))
+            d = self.download_request(request, Spider("foo", download_maxsize=1500))
             yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
             # As the error message is logged in the dataReceived callback, we
@@ -55,7 +58,7 @@ class Https2TestCase(Https11TestCase):
             # after closing the connection.
             d = defer.Deferred()
             d.addCallback(check)
-            reactor.callLater(.1, d.callback, logger)
+            reactor.callLater(0.1, d.callback, logger)
             yield d
 
     @defer.inlineCallbacks
@@ -64,88 +67,86 @@ class Https2TestCase(Https11TestCase):
         d = self.download_request(request, Spider("foo"))
         yield self.assertFailure(d, SchemeNotSupported)
 
-    def test_download_broken_content_cause_data_loss(self, url='broken'):
+    def test_download_broken_content_cause_data_loss(self, url="broken"):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
     def test_download_broken_chunked_content_cause_data_loss(self):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
-    def test_download_broken_content_allow_data_loss(self, url='broken'):
+    def test_download_broken_content_allow_data_loss(self, url="broken"):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
     def test_download_broken_chunked_content_allow_data_loss(self):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
-    def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
+    def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
     def test_concurrent_requests_same_domain(self):
-        spider = Spider('foo')
+        spider = Spider("foo")
 
-        request1 = Request(self.getURL('file'))
+        request1 = Request(self.getURL("file"))
         d1 = self.download_request(request1, spider)
         d1.addCallback(lambda r: r.body)
         d1.addCallback(self.assertEqual, b"0123456789")
 
-        request2 = Request(self.getURL('echo'), method='POST')
+        request2 = Request(self.getURL("echo"), method="POST")
         d2 = self.download_request(request2, spider)
-        d2.addCallback(lambda r: r.headers['Content-Length'])
+        d2.addCallback(lambda r: r.headers["Content-Length"])
         d2.addCallback(self.assertEqual, b"79")
 
         return defer.DeferredList([d1, d2])
 
     @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")
     def test_connect_request(self):
-        request = Request(self.getURL('file'), method='CONNECT')
-        d = self.download_request(request, Spider('foo'))
+        request = Request(self.getURL("file"), method="CONNECT")
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b'')
+        d.addCallback(self.assertEqual, b"")
         return d
 
     def test_custom_content_length_good(self):
-        request = Request(self.getURL('contentlength'))
+        request = Request(self.getURL("contentlength"))
         custom_content_length = str(len(request.body))
-        request.headers['Content-Length'] = custom_content_length
-        d = self.download_request(request, Spider('foo'))
+        request.headers["Content-Length"] = custom_content_length
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.text)
         d.addCallback(self.assertEqual, custom_content_length)
         return d
 
     def test_custom_content_length_bad(self):
-        request = Request(self.getURL('contentlength'))
+        request = Request(self.getURL("contentlength"))
         actual_content_length = str(len(request.body))
         bad_content_length = str(len(request.body) + 1)
-        request.headers['Content-Length'] = bad_content_length
+        request.headers["Content-Length"] = bad_content_length
         log = LogCapture()
-        d = self.download_request(request, Spider('foo'))
+        d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.text)
         d.addCallback(self.assertEqual, actual_content_length)
         d.addCallback(
             lambda _: log.check_present(
                 (
-                    'scrapy.core.http2.stream',
-                    'WARNING',
-                    f'Ignoring bad Content-Length header '
-                    f'{bad_content_length!r} of request {request}, sending '
-                    f'{actual_content_length!r} instead',
+                    "scrapy.core.http2.stream",
+                    "WARNING",
+                    f"Ignoring bad Content-Length header "
+                    f"{bad_content_length!r} of request {request}, sending "
+                    f"{actual_content_length!r} instead",
                 )
             )
         )
-        d.addCallback(
-            lambda _: log.uninstall()
-        )
+        d.addCallback(lambda _: log.uninstall())
         return d
 
     def test_duplicate_header(self):
-        request = Request(self.getURL('echo'))
-        header, value1, value2 = 'Custom-Header', 'foo', 'bar'
+        request = Request(self.getURL("echo"))
+        header, value1, value2 = "Custom-Header", "foo", "bar"
         request.headers.appendlist(header, value1)
         request.headers.appendlist(header, value2)
-        d = self.download_request(request, Spider('foo'))
-        d.addCallback(lambda r: json.loads(r.text)['headers'][header])
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: json.loads(r.text)["headers"][header])
         d.addCallback(self.assertEqual, [value1, value2])
         return d
 
@@ -163,8 +164,8 @@ class Https2WrongHostnameTestCase(Https2TestCase):
     # here we test that even if the server certificate is for another domain,
     # "www.example.com" in this case,
     # the tests still pass
-    keyfile = 'keys/example-com.key.pem'
-    certfile = 'keys/example-com.cert.pem'
+    keyfile = "keys/example-com.key.pem"
+    certfile = "keys/example-com.cert.pem"
 
 
 class Https2InvalidDNSId(Https2TestCase):
@@ -172,14 +173,14 @@ class Https2InvalidDNSId(Https2TestCase):
 
     def setUp(self):
         super().setUp()
-        self.host = '127.0.0.1'
+        self.host = "127.0.0.1"
 
 
 class Https2InvalidDNSPattern(Https2TestCase):
     """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
 
-    keyfile = 'keys/localhost.ip.key'
-    certfile = 'keys/localhost.ip.crt'
+    keyfile = "keys/localhost.ip.key"
+    certfile = "keys/localhost.ip.crt"
 
     def setUp(self):
         try:
@@ -195,19 +196,21 @@ class Https2InvalidDNSPattern(Https2TestCase):
 
 @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2CustomCiphers(Https11CustomCiphers):
-    scheme = 'https'
+    scheme = "https"
 
     @classmethod
     def setUpClass(cls):
         from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+
         cls.download_handler_cls = H2DownloadHandler
 
 
 class Http2MockServerTestCase(Http11MockServerTestCase):
     """HTTP 2.0 test case with MockServer"""
+
     settings_dict = {
-        'DOWNLOAD_HANDLERS': {
-            'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler'
+        "DOWNLOAD_HANDLERS": {
+            "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler"
         }
     }
 
@@ -215,28 +218,32 @@ class Http2MockServerTestCase(Http11MockServerTestCase):
 @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ProxyTestCase(Http11ProxyTestCase):
     # only used for HTTPS tests
-    keyfile = 'keys/localhost.key'
-    certfile = 'keys/localhost.crt'
+    keyfile = "keys/localhost.key"
+    certfile = "keys/localhost.crt"
 
-    scheme = 'https'
-    host = '127.0.0.1'
+    scheme = "https"
+    host = "127.0.0.1"
 
-    expected_http_proxy_request_body = b'/'
+    expected_http_proxy_request_body = b"/"
 
     @classmethod
     def setUpClass(cls):
         from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+
         cls.download_handler_cls = H2DownloadHandler
 
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
         self.port = reactor.listenSSL(
-            0, site,
+            0,
+            site,
             ssl_context_factory(self.keyfile, self.certfile),
-            interface=self.host
+            interface=self.host,
         )
         self.portno = self.port.getHost().port
-        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
+        self.download_handler = create_instance(
+            self.download_handler_cls, None, get_crawler()
+        )
         self.download_request = self.download_handler.download_request
 
     def getURL(self, path):
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 38be915f2..d8e377519 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -21,14 +21,14 @@ class ManagerTestCase(TestCase):
 
     def setUp(self):
         self.crawler = get_crawler(Spider, self.settings_dict)
-        self.spider = self.crawler._create_spider('foo')
+        self.spider = self.crawler._create_spider("foo")
         self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler)
         # some mw depends on stats collector
         self.crawler.stats.open_spider(self.spider)
         return self.mwman.open_spider(self.spider)
 
     def tearDown(self):
-        self.crawler.stats.close_spider(self.spider, '')
+        self.crawler.stats.close_spider(self.spider, "")
         return self.mwman.close_spider(self.spider)
 
     def _download(self, request, response=None):
@@ -57,7 +57,7 @@ class DefaultsTest(ManagerTestCase):
     """Tests default behavior with default settings"""
 
     def test_request_response(self):
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
         resp = Response(req.url, status=200)
         ret = self._download(req, resp)
         self.assertTrue(isinstance(ret, Response), "Non-response returned")
@@ -74,29 +74,41 @@ class DefaultsTest(ManagerTestCase):
             exceptions.IOError: Not a gzipped file
 
         """
-        req = Request('http://example.com')
-        body = b'<p>You are being redirected</p>'
-        resp = Response(req.url, status=302, body=body, headers={
-            'Content-Length': str(len(body)),
-            'Content-Type': 'text/html',
-            'Content-Encoding': 'gzip',
-            'Location': 'http://example.com/login',
-        })
+        req = Request("http://example.com")
+        body = b"<p>You are being redirected</p>"
+        resp = Response(
+            req.url,
+            status=302,
+            body=body,
+            headers={
+                "Content-Length": str(len(body)),
+                "Content-Type": "text/html",
+                "Content-Encoding": "gzip",
+                "Location": "http://example.com/login",
+            },
+        )
         ret = self._download(request=req, response=resp)
-        self.assertTrue(isinstance(ret, Request),
-                        f"Not redirected: {ret!r}")
-        self.assertEqual(to_bytes(ret.url), resp.headers['Location'],
-                         "Not redirected to location header")
+        self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}")
+        self.assertEqual(
+            to_bytes(ret.url),
+            resp.headers["Location"],
+            "Not redirected to location header",
+        )
 
     def test_200_and_invalid_gzipped_body_must_fail(self):
-        req = Request('http://example.com')
-        body = b'<p>You are being redirected</p>'
-        resp = Response(req.url, status=200, body=body, headers={
-            'Content-Length': str(len(body)),
-            'Content-Type': 'text/html',
-            'Content-Encoding': 'gzip',
-            'Location': 'http://example.com/login',
-        })
+        req = Request("http://example.com")
+        body = b"<p>You are being redirected</p>"
+        resp = Response(
+            req.url,
+            status=200,
+            body=body,
+            headers={
+                "Content-Length": str(len(body)),
+                "Content-Type": "text/html",
+                "Content-Encoding": "gzip",
+                "Location": "http://example.com/login",
+            },
+        )
         self.assertRaises(IOError, self._download, request=req, response=resp)
 
 
@@ -104,7 +116,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
     """Tests middleware returning a response from process_request."""
 
     def test_download_func_not_called(self):
-        resp = Response('http://example.com/index.html')
+        resp = Response("http://example.com/index.html")
 
         class ResponseMiddleware:
             def process_request(self, request, spider):
@@ -112,7 +124,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
 
         self.mwman._add_middleware(ResponseMiddleware())
 
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
         download_func = mock.MagicMock()
         dfd = self.mwman.download(download_func, req, self.spider)
         results = []
@@ -127,7 +139,7 @@ class ProcessRequestInvalidOutput(ManagerTestCase):
     """Invalid return value for process_request method should raise an exception"""
 
     def test_invalid_process_request(self):
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
 
         class InvalidProcessRequestMiddleware:
             def process_request(self, request, spider):
@@ -146,7 +158,7 @@ class ProcessResponseInvalidOutput(ManagerTestCase):
     """Invalid return value for process_response method should raise an exception"""
 
     def test_invalid_process_response(self):
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
 
         class InvalidProcessResponseMiddleware:
             def process_response(self, request, response, spider):
@@ -165,7 +177,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
     """Invalid return value for process_exception method should raise an exception"""
 
     def test_invalid_process_exception(self):
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
 
         class InvalidProcessExceptionMiddleware:
             def process_request(self, request, spider):
@@ -187,7 +199,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
     """Middlewares using Deferreds should work"""
 
     def test_deferred(self):
-        resp = Response('http://example.com/index.html')
+        resp = Response("http://example.com/index.html")
 
         class DeferredMiddleware:
             def cb(self, result):
@@ -200,7 +212,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
                 return d
 
         self.mwman._add_middleware(DeferredMiddleware())
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
         download_func = mock.MagicMock()
         dfd = self.mwman.download(download_func, req, self.spider)
         results = []
@@ -211,12 +223,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
         self.assertFalse(download_func.called)
 
 
-@mark.usefixtures('reactor_pytest')
+@mark.usefixtures("reactor_pytest")
 class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
-        resp = Response('http://example.com/index.html')
+        resp = Response("http://example.com/index.html")
 
         class CoroMiddleware:
             async def process_request(self, request, spider):
@@ -224,7 +236,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
                 return resp
 
         self.mwman._add_middleware(CoroMiddleware())
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
         download_func = mock.MagicMock()
         dfd = self.mwman.download(download_func, req, self.spider)
         results = []
@@ -236,7 +248,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
 
     @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
-        resp = Response('http://example.com/index.html')
+        resp = Response("http://example.com/index.html")
 
         class CoroMiddleware:
             async def process_request(self, request, spider):
@@ -245,7 +257,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
                 return result
 
         self.mwman._add_middleware(CoroMiddleware())
-        req = Request('http://example.com/index.html')
+        req = Request("http://example.com/index.html")
         download_func = mock.MagicMock()
         dfd = self.mwman.download(download_func, req, self.spider)
         results = []
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
index 5a56c9db2..6be107f6f 100644
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py
@@ -6,13 +6,13 @@ from scrapy.http import Request, HtmlResponse, Response
 from scrapy.utils.test import get_crawler
 
 
-__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl']
+__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"]
 
 
 class AjaxCrawlMiddlewareTest(unittest.TestCase):
     def setUp(self):
-        crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True})
-        self.spider = crawler._create_spider('foo')
+        crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True})
+        self.spider = crawler._create_spider("foo")
         self.mw = AjaxCrawlMiddleware.from_crawler(crawler)
 
     def _ajaxcrawlable_body(self):
@@ -24,37 +24,41 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase):
         return req, resp
 
     def test_non_get(self):
-        req, resp = self._req_resp('http://example.com/', {'method': 'HEAD'})
+        req, resp = self._req_resp("http://example.com/", {"method": "HEAD"})
         resp2 = self.mw.process_response(req, resp, self.spider)
         self.assertEqual(resp, resp2)
 
     def test_binary_response(self):
-        req = Request('http://example.com/')
-        resp = Response('http://example.com/', body=b'foobar\x00\x01\x02', request=req)
+        req = Request("http://example.com/")
+        resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req)
         resp2 = self.mw.process_response(req, resp, self.spider)
         self.assertIs(resp, resp2)
 
     def test_ajaxcrawl(self):
         req, resp = self._req_resp(
-            'http://example.com/',
-            {'meta': {'foo': 'bar'}},
-            {'body': self._ajaxcrawlable_body()}
+            "http://example.com/",
+            {"meta": {"foo": "bar"}},
+            {"body": self._ajaxcrawlable_body()},
         )
         req2 = self.mw.process_response(req, resp, self.spider)
-        self.assertEqual(req2.url, 'http://example.com/?_escaped_fragment_=')
-        self.assertEqual(req2.meta['foo'], 'bar')
+        self.assertEqual(req2.url, "http://example.com/?_escaped_fragment_=")
+        self.assertEqual(req2.meta["foo"], "bar")
 
     def test_ajaxcrawl_loop(self):
-        req, resp = self._req_resp('http://example.com/', {}, {'body': self._ajaxcrawlable_body()})
+        req, resp = self._req_resp(
+            "http://example.com/", {}, {"body": self._ajaxcrawlable_body()}
+        )
         req2 = self.mw.process_response(req, resp, self.spider)
         resp2 = HtmlResponse(req2.url, body=resp.body, request=req2)
         resp3 = self.mw.process_response(req2, resp2, self.spider)
 
         assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3)
-        self.assertEqual(resp3.request.url, 'http://example.com/?_escaped_fragment_=')
+        self.assertEqual(resp3.request.url, "http://example.com/?_escaped_fragment_=")
         assert resp3 is resp2
 
     def test_noncrawlable_body(self):
-        req, resp = self._req_resp('http://example.com/', {}, {'body': b'<html></html>'})
+        req, resp = self._req_resp(
+            "http://example.com/", {}, {"body": b"<html></html>"}
+        )
         resp2 = self.mw.process_response(req, resp, self.spider)
         self.assertIs(resp, resp2)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index dd835b9c9..812c003da 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -48,24 +48,18 @@ def _cookies_to_set_cookie_list(cookies):
         return []
     if isinstance(cookies, dict):
         cookies = ({"name": k, "value": v} for k, v in cookies.items())
-    return filter(
-        None,
-        (
-            _cookie_to_set_cookie_value(cookie)
-            for cookie in cookies
-        )
-    )
+    return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies))
 
 
 class CookiesMiddlewareTest(TestCase):
-
     def assertCookieValEqual(self, first, second, msg=None):
         def split_cookies(cookies):
             return sorted([s.strip() for s in to_bytes(cookies).split(b";")])
+
         return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
 
     def setUp(self):
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
         self.mw = CookiesMiddleware()
         self.redirect_middleware = RedirectMiddleware(settings=Settings())
 
@@ -74,346 +68,391 @@ class CookiesMiddlewareTest(TestCase):
         del self.redirect_middleware
 
     def test_basic(self):
-        req = Request('http://scrapytest.org/')
+        req = Request("http://scrapytest.org/")
         assert self.mw.process_request(req, self.spider) is None
-        assert 'Cookie' not in req.headers
+        assert "Cookie" not in req.headers
 
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
-        res = Response('http://scrapytest.org/', headers=headers)
+        headers = {"Set-Cookie": "C1=value1; path=/"}
+        res = Response("http://scrapytest.org/", headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
-        req2 = Request('http://scrapytest.org/sub1/')
+        req2 = Request("http://scrapytest.org/sub1/")
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertEqual(req2.headers.get('Cookie'), b"C1=value1")
+        self.assertEqual(req2.headers.get("Cookie"), b"C1=value1")
 
     def test_setting_false_cookies_enabled(self):
         self.assertRaises(
             NotConfigured,
             CookiesMiddleware.from_crawler,
-            get_crawler(settings_dict={'COOKIES_ENABLED': False})
+            get_crawler(settings_dict={"COOKIES_ENABLED": False}),
         )
 
     def test_setting_default_cookies_enabled(self):
         self.assertIsInstance(
-            CookiesMiddleware.from_crawler(get_crawler()),
-            CookiesMiddleware
+            CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware
         )
 
     def test_setting_true_cookies_enabled(self):
         self.assertIsInstance(
             CookiesMiddleware.from_crawler(
-                get_crawler(settings_dict={'COOKIES_ENABLED': True})
+                get_crawler(settings_dict={"COOKIES_ENABLED": True})
             ),
-            CookiesMiddleware
+            CookiesMiddleware,
         )
 
     def test_setting_enabled_cookies_debug(self):
-        crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
+        crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture(
-            'scrapy.downloadermiddlewares.cookies',
+            "scrapy.downloadermiddlewares.cookies",
             propagate=False,
             level=logging.DEBUG,
         ) as log:
-            req = Request('http://scrapytest.org/')
-            res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
+            req = Request("http://scrapytest.org/")
+            res = Response(
+                "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
+            )
             mw.process_response(req, res, crawler.spider)
-            req2 = Request('http://scrapytest.org/sub1/')
+            req2 = Request("http://scrapytest.org/sub1/")
             mw.process_request(req2, crawler.spider)
 
             log.check(
-                ('scrapy.downloadermiddlewares.cookies',
-                 'DEBUG',
-                 'Received cookies from: <200 http://scrapytest.org/>\n'
-                 'Set-Cookie: C1=value1; path=/\n'),
-                ('scrapy.downloadermiddlewares.cookies',
-                 'DEBUG',
-                 'Sending cookies to: <GET http://scrapytest.org/sub1/>\n'
-                 'Cookie: C1=value1\n'),
+                (
+                    "scrapy.downloadermiddlewares.cookies",
+                    "DEBUG",
+                    "Received cookies from: <200 http://scrapytest.org/>\n"
+                    "Set-Cookie: C1=value1; path=/\n",
+                ),
+                (
+                    "scrapy.downloadermiddlewares.cookies",
+                    "DEBUG",
+                    "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
+                    "Cookie: C1=value1\n",
+                ),
             )
 
     def test_setting_disabled_cookies_debug(self):
-        crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
+        crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False})
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture(
-            'scrapy.downloadermiddlewares.cookies',
+            "scrapy.downloadermiddlewares.cookies",
             propagate=False,
             level=logging.DEBUG,
         ) as log:
-            req = Request('http://scrapytest.org/')
-            res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
+            req = Request("http://scrapytest.org/")
+            res = Response(
+                "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
+            )
             mw.process_response(req, res, crawler.spider)
-            req2 = Request('http://scrapytest.org/sub1/')
+            req2 = Request("http://scrapytest.org/sub1/")
             mw.process_request(req2, crawler.spider)
 
             log.check()
 
     def test_do_not_break_on_non_utf8_header(self):
-        req = Request('http://scrapytest.org/')
+        req = Request("http://scrapytest.org/")
         assert self.mw.process_request(req, self.spider) is None
-        assert 'Cookie' not in req.headers
+        assert "Cookie" not in req.headers
 
-        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'}
-        res = Response('http://scrapytest.org/', headers=headers)
+        headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"}
+        res = Response("http://scrapytest.org/", headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
-        req2 = Request('http://scrapytest.org/sub1/')
+        req2 = Request("http://scrapytest.org/sub1/")
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertIn('Cookie', req2.headers)
+        self.assertIn("Cookie", req2.headers)
 
     def test_dont_merge_cookies(self):
         # merge some cookies into jar
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
-        req = Request('http://scrapytest.org/')
-        res = Response('http://scrapytest.org/', headers=headers)
+        headers = {"Set-Cookie": "C1=value1; path=/"}
+        req = Request("http://scrapytest.org/")
+        res = Response("http://scrapytest.org/", headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
         # test Cookie header is not seted to request
-        req = Request('http://scrapytest.org/dontmerge', meta={'dont_merge_cookies': 1})
+        req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1})
         assert self.mw.process_request(req, self.spider) is None
-        assert 'Cookie' not in req.headers
+        assert "Cookie" not in req.headers
 
         # check that returned cookies are not merged back to jar
         res = Response(
-            'http://scrapytest.org/dontmerge',
-            headers={'Set-Cookie': 'dont=mergeme; path=/'},
+            "http://scrapytest.org/dontmerge",
+            headers={"Set-Cookie": "dont=mergeme; path=/"},
         )
         assert self.mw.process_response(req, res, self.spider) is res
 
         # check that cookies are merged back
-        req = Request('http://scrapytest.org/mergeme')
+        req = Request("http://scrapytest.org/mergeme")
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req.headers.get("Cookie"), b"C1=value1")
 
         # check that cookies are merged when dont_merge_cookies is passed as 0
-        req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0})
+        req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req.headers.get("Cookie"), b"C1=value1")
 
     def test_complex_cookies(self):
         # merge some cookies into jar
         cookies = [
-            {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
-            {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
-            {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
-            {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
+            {
+                "name": "C1",
+                "value": "value1",
+                "path": "/foo",
+                "domain": "scrapytest.org",
+            },
+            {
+                "name": "C2",
+                "value": "value2",
+                "path": "/bar",
+                "domain": "scrapytest.org",
+            },
+            {
+                "name": "C3",
+                "value": "value3",
+                "path": "/foo",
+                "domain": "scrapytest.org",
+            },
+            {"name": "C4", "value": "value4", "path": "/foo", "domain": "scrapy.org"},
         ]
 
-        req = Request('http://scrapytest.org/', cookies=cookies)
+        req = Request("http://scrapytest.org/", cookies=cookies)
         self.mw.process_request(req, self.spider)
 
         # embed C1 and C3 for scrapytest.org/foo
-        req = Request('http://scrapytest.org/foo')
+        req = Request("http://scrapytest.org/foo")
         self.mw.process_request(req, self.spider)
-        assert req.headers.get('Cookie') in (b'C1=value1; C3=value3', b'C3=value3; C1=value1')
+        assert req.headers.get("Cookie") in (
+            b"C1=value1; C3=value3",
+            b"C3=value3; C1=value1",
+        )
 
         # embed C2 for scrapytest.org/bar
-        req = Request('http://scrapytest.org/bar')
+        req = Request("http://scrapytest.org/bar")
         self.mw.process_request(req, self.spider)
-        self.assertEqual(req.headers.get('Cookie'), b'C2=value2')
+        self.assertEqual(req.headers.get("Cookie"), b"C2=value2")
 
         # embed nothing for scrapytest.org/baz
-        req = Request('http://scrapytest.org/baz')
+        req = Request("http://scrapytest.org/baz")
         self.mw.process_request(req, self.spider)
-        assert 'Cookie' not in req.headers
+        assert "Cookie" not in req.headers
 
     def test_merge_request_cookies(self):
-        req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'})
+        req = Request("http://scrapytest.org/", cookies={"galleta": "salada"})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
+        self.assertEqual(req.headers.get("Cookie"), b"galleta=salada")
 
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
-        res = Response('http://scrapytest.org/', headers=headers)
+        headers = {"Set-Cookie": "C1=value1; path=/"}
+        res = Response("http://scrapytest.org/", headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
-        req2 = Request('http://scrapytest.org/sub1/')
+        req2 = Request("http://scrapytest.org/sub1/")
         assert self.mw.process_request(req2, self.spider) is None
 
-        self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada")
+        self.assertCookieValEqual(
+            req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
+        )
 
     def test_cookiejar_key(self):
         req = Request(
-            'http://scrapytest.org/',
-            cookies={'galleta': 'salada'},
-            meta={'cookiejar': "store1"},
+            "http://scrapytest.org/",
+            cookies={"galleta": "salada"},
+            meta={"cookiejar": "store1"},
         )
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
+        self.assertEqual(req.headers.get("Cookie"), b"galleta=salada")
 
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
-        res = Response('http://scrapytest.org/', headers=headers, request=req)
+        headers = {"Set-Cookie": "C1=value1; path=/"}
+        res = Response("http://scrapytest.org/", headers=headers, request=req)
         assert self.mw.process_response(req, res, self.spider) is res
 
-        req2 = Request('http://scrapytest.org/', meta=res.meta)
+        req2 = Request("http://scrapytest.org/", meta=res.meta)
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada')
+        self.assertCookieValEqual(
+            req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
+        )
 
         req3 = Request(
-            'http://scrapytest.org/',
-            cookies={'galleta': 'dulce'},
-            meta={'cookiejar': "store2"},
+            "http://scrapytest.org/",
+            cookies={"galleta": "dulce"},
+            meta={"cookiejar": "store2"},
         )
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce')
+        self.assertEqual(req3.headers.get("Cookie"), b"galleta=dulce")
 
-        headers = {'Set-Cookie': 'C2=value2; path=/'}
-        res2 = Response('http://scrapytest.org/', headers=headers, request=req3)
+        headers = {"Set-Cookie": "C2=value2; path=/"}
+        res2 = Response("http://scrapytest.org/", headers=headers, request=req3)
         assert self.mw.process_response(req3, res2, self.spider) is res2
 
-        req4 = Request('http://scrapytest.org/', meta=res2.meta)
+        req4 = Request("http://scrapytest.org/", meta=res2.meta)
         assert self.mw.process_request(req4, self.spider) is None
-        self.assertCookieValEqual(req4.headers.get('Cookie'), b'C2=value2; galleta=dulce')
+        self.assertCookieValEqual(
+            req4.headers.get("Cookie"), b"C2=value2; galleta=dulce"
+        )
 
         # cookies from hosts with port
-        req5_1 = Request('http://scrapytest.org:1104/')
+        req5_1 = Request("http://scrapytest.org:1104/")
         assert self.mw.process_request(req5_1, self.spider) is None
 
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
-        res5_1 = Response('http://scrapytest.org:1104/', headers=headers, request=req5_1)
+        headers = {"Set-Cookie": "C1=value1; path=/"}
+        res5_1 = Response(
+            "http://scrapytest.org:1104/", headers=headers, request=req5_1
+        )
         assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1
 
-        req5_2 = Request('http://scrapytest.org:1104/some-redirected-path')
+        req5_2 = Request("http://scrapytest.org:1104/some-redirected-path")
         assert self.mw.process_request(req5_2, self.spider) is None
-        self.assertEqual(req5_2.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req5_2.headers.get("Cookie"), b"C1=value1")
 
-        req5_3 = Request('http://scrapytest.org/some-redirected-path')
+        req5_3 = Request("http://scrapytest.org/some-redirected-path")
         assert self.mw.process_request(req5_3, self.spider) is None
-        self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req5_3.headers.get("Cookie"), b"C1=value1")
 
         # skip cookie retrieval for not http request
-        req6 = Request('file:///scrapy/sometempfile')
+        req6 = Request("file:///scrapy/sometempfile")
         assert self.mw.process_request(req6, self.spider) is None
-        self.assertEqual(req6.headers.get('Cookie'), None)
+        self.assertEqual(req6.headers.get("Cookie"), None)
 
     def test_local_domain(self):
-        request = Request("http://example-host/", cookies={'currencyCookie': 'USD'})
+        request = Request("http://example-host/", cookies={"currencyCookie": "USD"})
         assert self.mw.process_request(request, self.spider) is None
-        self.assertIn('Cookie', request.headers)
-        self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
+        self.assertIn("Cookie", request.headers)
+        self.assertEqual(b"currencyCookie=USD", request.headers["Cookie"])
 
     @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_keep_cookie_from_default_request_headers_middleware(self):
-        DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
+        DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty")
         mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
         # overwrite with values from 'cookies' request argument
-        req1 = Request('http://example.org', cookies={'default': 'something'})
+        req1 = Request("http://example.org", cookies={"default": "something"})
         assert mw_default_headers.process_request(req1, self.spider) is None
         assert self.mw.process_request(req1, self.spider) is None
-        self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty')
+        self.assertCookieValEqual(
+            req1.headers["Cookie"], b"default=something; asdf=qwerty"
+        )
         # keep both
-        req2 = Request('http://example.com', cookies={'a': 'b'})
+        req2 = Request("http://example.com", cookies={"a": "b"})
         assert mw_default_headers.process_request(req2, self.spider) is None
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
+        self.assertCookieValEqual(
+            req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty"
+        )
 
     @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_keep_cookie_header(self):
         # keep only cookies from 'Cookie' request header
-        req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
+        req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"})
         assert self.mw.process_request(req1, self.spider) is None
-        self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d')
+        self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d")
         # keep cookies from both 'Cookie' request header and 'cookies' keyword
-        req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'})
+        req2 = Request(
+            "http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"}
+        )
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f')
+        self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f")
         # overwrite values from 'Cookie' request header with 'cookies' keyword
         req3 = Request(
-            'http://scrapytest.org',
-            headers={'Cookie': 'a=b; c=d'},
-            cookies={'a': 'new', 'e': 'f'},
+            "http://scrapytest.org",
+            headers={"Cookie": "a=b; c=d"},
+            cookies={"a": "new", "e": "f"},
         )
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f')
+        self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f")
 
     def test_request_cookies_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
+        req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")})
         assert self.mw.process_request(req1, self.spider) is None
-        self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
+        self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
+        req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")})
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
+        self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")
 
         # 3) String
-        req3 = Request('http://example.org', cookies={'a': 'á'})
+        req3 = Request("http://example.org", cookies={"a": "á"})
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
+        self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
 
     @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
+        req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")})
         assert self.mw.process_request(req1, self.spider) is None
-        self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
+        self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
+        req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")})
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
+        self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")
 
         # 3) String
-        req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
+        req3 = Request("http://example.org", headers={"Cookie": "a=á"})
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
+        self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
 
     def test_invalid_cookies(self):
         """
         Invalid cookies are logged as warnings and discarded
         """
         with LogCapture(
-            'scrapy.downloadermiddlewares.cookies',
+            "scrapy.downloadermiddlewares.cookies",
             propagate=False,
             level=logging.INFO,
         ) as lc:
-            cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}]
-            req1 = Request('http://example.org/1', cookies=cookies1)
+            cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}]
+            req1 = Request("http://example.org/1", cookies=cookies1)
             assert self.mw.process_request(req1, self.spider) is None
-            cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}]
-            req2 = Request('http://example.org/2', cookies=cookies2)
+            cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}]
+            req2 = Request("http://example.org/2", cookies=cookies2)
             assert self.mw.process_request(req2, self.spider) is None
-            cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}]
-            req3 = Request('http://example.org/3', cookies=cookies3)
+            cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}]
+            req3 = Request("http://example.org/3", cookies=cookies3)
             assert self.mw.process_request(req3, self.spider) is None
             lc.check(
-                ("scrapy.downloadermiddlewares.cookies",
-                 "WARNING",
-                 "Invalid cookie found in request <GET http://example.org/1>:"
-                 " {'value': 'bar'} ('name' is missing)"),
-                ("scrapy.downloadermiddlewares.cookies",
-                 "WARNING",
-                 "Invalid cookie found in request <GET http://example.org/2>:"
-                 " {'name': 'foo'} ('value' is missing)"),
-                ("scrapy.downloadermiddlewares.cookies",
-                 "WARNING",
-                 "Invalid cookie found in request <GET http://example.org/3>:"
-                 " {'name': 'foo', 'value': None} ('value' is missing)"),
+                (
+                    "scrapy.downloadermiddlewares.cookies",
+                    "WARNING",
+                    "Invalid cookie found in request <GET http://example.org/1>:"
+                    " {'value': 'bar'} ('name' is missing)",
+                ),
+                (
+                    "scrapy.downloadermiddlewares.cookies",
+                    "WARNING",
+                    "Invalid cookie found in request <GET http://example.org/2>:"
+                    " {'name': 'foo'} ('value' is missing)",
+                ),
+                (
+                    "scrapy.downloadermiddlewares.cookies",
+                    "WARNING",
+                    "Invalid cookie found in request <GET http://example.org/3>:"
+                    " {'name': 'foo', 'value': None} ('value' is missing)",
+                ),
             )
-        self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
-        self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')
-        self.assertCookieValEqual(req3.headers['Cookie'], 'key=')
+        self.assertCookieValEqual(req1.headers["Cookie"], "key=value1")
+        self.assertCookieValEqual(req2.headers["Cookie"], "key=value2")
+        self.assertCookieValEqual(req3.headers["Cookie"], "key=")
 
     def test_primitive_type_cookies(self):
         # Boolean
-        req1 = Request('http://example.org', cookies={'a': True})
+        req1 = Request("http://example.org", cookies={"a": True})
         assert self.mw.process_request(req1, self.spider) is None
-        self.assertCookieValEqual(req1.headers['Cookie'], b'a=True')
+        self.assertCookieValEqual(req1.headers["Cookie"], b"a=True")
 
         # Float
-        req2 = Request('http://example.org', cookies={'a': 9.5})
+        req2 = Request("http://example.org", cookies={"a": 9.5})
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5')
+        self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5")
 
         # Integer
-        req3 = Request('http://example.org', cookies={'a': 10})
+        req3 = Request("http://example.org", cookies={"a": 10})
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertCookieValEqual(req3.headers['Cookie'], b'a=10')
+        self.assertCookieValEqual(req3.headers["Cookie"], b"a=10")
 
         # String
-        req4 = Request('http://example.org', cookies={'a': 'b'})
+        req4 = Request("http://example.org", cookies={"a": "b"})
         assert self.mw.process_request(req4, self.spider) is None
-        self.assertCookieValEqual(req4.headers['Cookie'], b'a=b')
+        self.assertCookieValEqual(req4.headers["Cookie"], b"a=b")
 
     def _test_cookie_redirect(
         self,
@@ -423,22 +462,22 @@ class CookiesMiddlewareTest(TestCase):
         cookies1,
         cookies2,
     ):
-        input_cookies = {'a': 'b'}
+        input_cookies = {"a": "b"}
 
         if not isinstance(source, dict):
-            source = {'url': source}
+            source = {"url": source}
         if not isinstance(target, dict):
-            target = {'url': target}
-        target.setdefault('status', 301)
+            target = {"url": target}
+        target.setdefault("status", 301)
 
         request1 = Request(cookies=input_cookies, **source)
         self.mw.process_request(request1, self.spider)
-        cookies = request1.headers.get('Cookie')
+        cookies = request1.headers.get("Cookie")
         self.assertEqual(cookies, b"a=b" if cookies1 else None)
 
         response = Response(
             headers={
-                'Location': target['url'],
+                "Location": target["url"],
             },
             **target,
         )
@@ -455,37 +494,37 @@ class CookiesMiddlewareTest(TestCase):
         self.assertIsInstance(request2, Request)
 
         self.mw.process_request(request2, self.spider)
-        cookies = request2.headers.get('Cookie')
+        cookies = request2.headers.get("Cookie")
         self.assertEqual(cookies, b"a=b" if cookies2 else None)
 
     def test_cookie_redirect_same_domain(self):
         self._test_cookie_redirect(
-            'https://toscrape.com',
-            'https://toscrape.com',
+            "https://toscrape.com",
+            "https://toscrape.com",
             cookies1=True,
             cookies2=True,
         )
 
     def test_cookie_redirect_same_domain_forcing_get(self):
         self._test_cookie_redirect(
-            'https://toscrape.com',
-            {'url': 'https://toscrape.com', 'status': 302},
+            "https://toscrape.com",
+            {"url": "https://toscrape.com", "status": 302},
             cookies1=True,
             cookies2=True,
         )
 
     def test_cookie_redirect_different_domain(self):
         self._test_cookie_redirect(
-            'https://toscrape.com',
-            'https://example.com',
+            "https://toscrape.com",
+            "https://example.com",
             cookies1=True,
             cookies2=False,
         )
 
     def test_cookie_redirect_different_domain_forcing_get(self):
         self._test_cookie_redirect(
-            'https://toscrape.com',
-            {'url': 'https://example.com', 'status': 302},
+            "https://toscrape.com",
+            {"url": "https://example.com", "status": 302},
             cookies1=True,
             cookies2=False,
         )
@@ -514,16 +553,16 @@ class CookiesMiddlewareTest(TestCase):
                   because the middleware empties the header from every request.
         """
         if not isinstance(source, dict):
-            source = {'url': source}
+            source = {"url": source}
         if not isinstance(target, dict):
-            target = {'url': target}
-        target.setdefault('status', 301)
+            target = {"url": target}
+        target.setdefault("status", 301)
 
-        request1 = Request(headers={'Cookie': b'a=b'}, **source)
+        request1 = Request(headers={"Cookie": b"a=b"}, **source)
 
         response = Response(
             headers={
-                'Location': target['url'],
+                "Location": target["url"],
             },
             **target,
         )
@@ -535,34 +574,34 @@ class CookiesMiddlewareTest(TestCase):
         )
         self.assertIsInstance(request2, Request)
 
-        cookies = request2.headers.get('Cookie')
+        cookies = request2.headers.get("Cookie")
         self.assertEqual(cookies, b"a=b" if cookies2 else None)
 
     def test_cookie_header_redirect_same_domain(self):
         self._test_cookie_header_redirect(
-            'https://toscrape.com',
-            'https://toscrape.com',
+            "https://toscrape.com",
+            "https://toscrape.com",
             cookies2=True,
         )
 
     def test_cookie_header_redirect_same_domain_forcing_get(self):
         self._test_cookie_header_redirect(
-            'https://toscrape.com',
-            {'url': 'https://toscrape.com', 'status': 302},
+            "https://toscrape.com",
+            {"url": "https://toscrape.com", "status": 302},
             cookies2=True,
         )
 
     def test_cookie_header_redirect_different_domain(self):
         self._test_cookie_header_redirect(
-            'https://toscrape.com',
-            'https://example.com',
+            "https://toscrape.com",
+            "https://example.com",
             cookies2=False,
         )
 
     def test_cookie_header_redirect_different_domain_forcing_get(self):
         self._test_cookie_header_redirect(
-            'https://toscrape.com',
-            {'url': 'https://example.com', 'status': 302},
+            "https://toscrape.com",
+            {"url": "https://example.com", "status": 302},
             cookies2=False,
         )
 
@@ -577,54 +616,54 @@ class CookiesMiddlewareTest(TestCase):
     ):
         input_cookies = [
             {
-                'name': 'a',
-                'value': 'b',
-                'domain': domain,
+                "name": "a",
+                "value": "b",
+                "domain": domain,
             }
         ]
 
         request1 = Request(url1, cookies=input_cookies)
         self.mw.process_request(request1, self.spider)
-        cookies = request1.headers.get('Cookie')
+        cookies = request1.headers.get("Cookie")
         self.assertEqual(cookies, b"a=b" if cookies1 else None)
 
         request2 = Request(url2)
         self.mw.process_request(request2, self.spider)
-        cookies = request2.headers.get('Cookie')
+        cookies = request2.headers.get("Cookie")
         self.assertEqual(cookies, b"a=b" if cookies2 else None)
 
     def test_user_set_cookie_domain_suffix_private(self):
         self._test_user_set_cookie_domain_followup(
-            'https://books.toscrape.com',
-            'https://quotes.toscrape.com',
-            'toscrape.com',
+            "https://books.toscrape.com",
+            "https://quotes.toscrape.com",
+            "toscrape.com",
             cookies1=True,
             cookies2=True,
         )
 
     def test_user_set_cookie_domain_suffix_public_period(self):
         self._test_user_set_cookie_domain_followup(
-            'https://foo.co.uk',
-            'https://bar.co.uk',
-            'co.uk',
+            "https://foo.co.uk",
+            "https://bar.co.uk",
+            "co.uk",
             cookies1=False,
             cookies2=False,
         )
 
     def test_user_set_cookie_domain_suffix_public_private(self):
         self._test_user_set_cookie_domain_followup(
-            'https://foo.blogspot.com',
-            'https://bar.blogspot.com',
-            'blogspot.com',
+            "https://foo.blogspot.com",
+            "https://bar.blogspot.com",
+            "blogspot.com",
             cookies1=False,
             cookies2=False,
         )
 
     def test_user_set_cookie_domain_public_period(self):
         self._test_user_set_cookie_domain_followup(
-            'https://co.uk',
-            'https://co.uk',
-            'co.uk',
+            "https://co.uk",
+            "https://co.uk",
+            "co.uk",
             cookies1=True,
             cookies2=True,
         )
@@ -642,14 +681,14 @@ class CookiesMiddlewareTest(TestCase):
 
         input_cookies = [
             {
-                'name': 'a',
-                'value': 'b',
-                'domain': domain,
+                "name": "a",
+                "value": "b",
+                "domain": domain,
             }
         ]
 
         headers = {
-            'Set-Cookie': _cookies_to_set_cookie_list(input_cookies),
+            "Set-Cookie": _cookies_to_set_cookie_list(input_cookies),
         }
         response = Response(url1, status=200, headers=headers)
         self.assertEqual(
@@ -659,37 +698,37 @@ class CookiesMiddlewareTest(TestCase):
 
         request2 = Request(url2)
         self.mw.process_request(request2, self.spider)
-        actual_cookies = request2.headers.get('Cookie')
+        actual_cookies = request2.headers.get("Cookie")
         self.assertEqual(actual_cookies, b"a=b" if cookies else None)
 
     def test_server_set_cookie_domain_suffix_private(self):
         self._test_server_set_cookie_domain_followup(
-            'https://books.toscrape.com',
-            'https://quotes.toscrape.com',
-            'toscrape.com',
+            "https://books.toscrape.com",
+            "https://quotes.toscrape.com",
+            "toscrape.com",
             cookies=True,
         )
 
     def test_server_set_cookie_domain_suffix_public_period(self):
         self._test_server_set_cookie_domain_followup(
-            'https://foo.co.uk',
-            'https://bar.co.uk',
-            'co.uk',
+            "https://foo.co.uk",
+            "https://bar.co.uk",
+            "co.uk",
             cookies=False,
         )
 
     def test_server_set_cookie_domain_suffix_public_private(self):
         self._test_server_set_cookie_domain_followup(
-            'https://foo.blogspot.com',
-            'https://bar.blogspot.com',
-            'blogspot.com',
+            "https://foo.blogspot.com",
+            "https://bar.blogspot.com",
+            "blogspot.com",
             cookies=False,
         )
 
     def test_server_set_cookie_domain_public_period(self):
         self._test_server_set_cookie_domain_followup(
-            'https://co.uk',
-            'https://co.uk',
-            'co.uk',
+            "https://co.uk",
+            "https://co.uk",
+            "co.uk",
             cookies=True,
         )
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 1c4cae6d1..16ae9ed75 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -7,39 +7,39 @@ from tests import get_testdata
 
 
 def _test_data(formats):
-    uncompressed_body = get_testdata('compressed', 'feed-sample1.xml')
+    uncompressed_body = get_testdata("compressed", "feed-sample1.xml")
     test_responses = {}
     for format in formats:
-        body = get_testdata('compressed', 'feed-sample1.' + format)
-        test_responses[format] = Response('http://foo.com/bar', body=body)
+        body = get_testdata("compressed", "feed-sample1." + format)
+        test_responses[format] = Response("http://foo.com/bar", body=body)
     return uncompressed_body, test_responses
 
 
 class DecompressionMiddlewareTest(TestCase):
 
-    test_formats = ['tar', 'xml.bz2', 'xml.gz', 'zip']
+    test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]
     uncompressed_body, test_responses = _test_data(test_formats)
 
     def setUp(self):
         self.mw = DecompressionMiddleware()
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
 
     def test_known_compression_formats(self):
         for fmt in self.test_formats:
             rsp = self.test_responses[fmt]
             new = self.mw.process_response(None, rsp, self.spider)
-            error_msg = f'Failed {fmt}, response type {type(new).__name__}'
+            error_msg = f"Failed {fmt}, response type {type(new).__name__}"
             assert isinstance(new, XmlResponse), error_msg
             assert_samelines(self, new.body, self.uncompressed_body, fmt)
 
     def test_plain_response(self):
-        rsp = Response(url='http://test.com', body=self.uncompressed_body)
+        rsp = Response(url="http://test.com", body=self.uncompressed_body)
         new = self.mw.process_response(None, rsp, self.spider)
         assert new is rsp
         assert_samelines(self, new.body, rsp.body)
 
     def test_empty_response(self):
-        rsp = Response(url='http://test.com', body=b'')
+        rsp = Response(url="http://test.com", body=b"")
         new = self.mw.process_response(None, rsp, self.spider)
         assert new is rsp
         assert not rsp.body
@@ -49,5 +49,5 @@ class DecompressionMiddlewareTest(TestCase):
         del self.mw
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index 6a31dfcf8..601e85799 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -8,27 +8,26 @@ from scrapy.utils.python import to_bytes
 
 
 class TestDefaultHeadersMiddleware(TestCase):
-
     def get_defaults_spider_mw(self):
         crawler = get_crawler(Spider)
-        spider = crawler._create_spider('foo')
+        spider = crawler._create_spider("foo")
         defaults = {
             to_bytes(k): [to_bytes(v)]
-            for k, v in crawler.settings.get('DEFAULT_REQUEST_HEADERS').items()
+            for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
         }
         return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
 
     def test_process_request(self):
         defaults, spider, mw = self.get_defaults_spider_mw()
-        req = Request('http://www.scrapytest.org')
+        req = Request("http://www.scrapytest.org")
         mw.process_request(req, spider)
         self.assertEqual(req.headers, defaults)
 
     def test_update_headers(self):
         defaults, spider, mw = self.get_defaults_spider_mw()
-        headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
-        bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']}
-        req = Request('http://www.scrapytest.org', headers=headers)
+        headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
+        bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
+        req = Request("http://www.scrapytest.org", headers=headers)
         self.assertEqual(req.headers, bytes_headers)
 
         mw.process_request(req, spider)
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index 586bdc0d1..8d2b821b0 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -7,36 +7,35 @@ from scrapy.utils.test import get_crawler
 
 
 class DownloadTimeoutMiddlewareTest(unittest.TestCase):
-
     def get_request_spider_mw(self, settings=None):
         crawler = get_crawler(Spider, settings)
-        spider = crawler._create_spider('foo')
-        request = Request('http://scrapytest.org/')
+        spider = crawler._create_spider("foo")
+        request = Request("http://scrapytest.org/")
         return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
 
     def test_default_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw()
         mw.spider_opened(spider)
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta.get('download_timeout'), 180)
+        self.assertEqual(req.meta.get("download_timeout"), 180)
 
     def test_string_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'})
+        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
         mw.spider_opened(spider)
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta.get('download_timeout'), 20.1)
+        self.assertEqual(req.meta.get("download_timeout"), 20.1)
 
     def test_spider_has_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw()
         spider.download_timeout = 2
         mw.spider_opened(spider)
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta.get('download_timeout'), 2)
+        self.assertEqual(req.meta.get("download_timeout"), 2)
 
     def test_request_has_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw()
         spider.download_timeout = 2
         mw.spider_opened(spider)
-        req.meta['download_timeout'] = 1
+        req.meta["download_timeout"] = 1
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta.get('download_timeout'), 1)
+        self.assertEqual(req.meta.get("download_timeout"), 1)
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index b9f3e24a4..1320bded2 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -10,108 +10,106 @@ from scrapy.spiders import Spider
 
 
 class TestSpiderLegacy(Spider):
-    http_user = 'foo'
-    http_pass = 'bar'
+    http_user = "foo"
+    http_pass = "bar"
 
 
 class TestSpider(Spider):
-    http_user = 'foo'
-    http_pass = 'bar'
-    http_auth_domain = 'example.com'
+    http_user = "foo"
+    http_pass = "bar"
+    http_auth_domain = "example.com"
 
 
 class TestSpiderAny(Spider):
-    http_user = 'foo'
-    http_pass = 'bar'
+    http_user = "foo"
+    http_pass = "bar"
     http_auth_domain = None
 
 
 class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
-
     def setUp(self):
-        self.spider = TestSpiderLegacy('foo')
+        self.spider = TestSpiderLegacy("foo")
 
     def test_auth(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
+        ):
             mw = HttpAuthMiddleware()
             mw.spider_opened(self.spider)
 
         # initial request, sets the domain and sends the header
-        req = Request('http://example.com/')
+        req = Request("http://example.com/")
         assert mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
 
         # subsequent request to the same domain, should send the header
-        req = Request('http://example.com/')
+        req = Request("http://example.com/")
         assert mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
 
         # subsequent request to a different domain, shouldn't send the header
-        req = Request('http://example-noauth.com/')
+        req = Request("http://example-noauth.com/")
         assert mw.process_request(req, self.spider) is None
-        self.assertNotIn('Authorization', req.headers)
+        self.assertNotIn("Authorization", req.headers)
 
     def test_auth_already_set(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
+        ):
             mw = HttpAuthMiddleware()
             mw.spider_opened(self.spider)
-        req = Request('http://example.com/',
-                      headers=dict(Authorization='Digest 123'))
+        req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
         assert mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], b'Digest 123')
+        self.assertEqual(req.headers["Authorization"], b"Digest 123")
 
 
 class HttpAuthMiddlewareTest(unittest.TestCase):
-
     def setUp(self):
         self.mw = HttpAuthMiddleware()
-        self.spider = TestSpider('foo')
+        self.spider = TestSpider("foo")
         self.mw.spider_opened(self.spider)
 
     def tearDown(self):
         del self.mw
 
     def test_no_auth(self):
-        req = Request('http://example-noauth.com/')
+        req = Request("http://example-noauth.com/")
         assert self.mw.process_request(req, self.spider) is None
-        self.assertNotIn('Authorization', req.headers)
+        self.assertNotIn("Authorization", req.headers)
 
     def test_auth_domain(self):
-        req = Request('http://example.com/')
+        req = Request("http://example.com/")
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
 
     def test_auth_subdomain(self):
-        req = Request('http://foo.example.com/')
+        req = Request("http://foo.example.com/")
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
 
     def test_auth_already_set(self):
-        req = Request('http://example.com/',
-                      headers=dict(Authorization='Digest 123'))
+        req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], b'Digest 123')
+        self.assertEqual(req.headers["Authorization"], b"Digest 123")
 
 
 class HttpAuthAnyMiddlewareTest(unittest.TestCase):
-
     def setUp(self):
         self.mw = HttpAuthMiddleware()
-        self.spider = TestSpiderAny('foo')
+        self.spider = TestSpiderAny("foo")
         self.mw.spider_opened(self.spider)
 
     def tearDown(self):
         del self.mw
 
     def test_auth(self):
-        req = Request('http://example.com/')
+        req = Request("http://example.com/")
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
 
     def test_auth_already_set(self):
-        req = Request('http://example.com/',
-                      headers=dict(Authorization='Digest 123'))
+        req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], b'Digest 123')
+        self.assertEqual(req.headers["Authorization"], b"Digest 123")
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index b3d8264ba..caa89b6bd 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -15,36 +15,37 @@ from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
 
 class _BaseTest(unittest.TestCase):
 
-    storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage'
-    policy_class = 'scrapy.extensions.httpcache.RFC2616Policy'
+    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+    policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
     def setUp(self):
         self.yesterday = email.utils.formatdate(time.time() - 86400)
         self.today = email.utils.formatdate()
         self.tomorrow = email.utils.formatdate(time.time() + 86400)
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('example.com')
+        self.spider = self.crawler._create_spider("example.com")
         self.tmpdir = tempfile.mkdtemp()
-        self.request = Request('http://www.example.com',
-                               headers={'User-Agent': 'test'})
-        self.response = Response('http://www.example.com',
-                                 headers={'Content-Type': 'text/html'},
-                                 body=b'test body',
-                                 status=202)
+        self.request = Request("http://www.example.com", headers={"User-Agent": "test"})
+        self.response = Response(
+            "http://www.example.com",
+            headers={"Content-Type": "text/html"},
+            body=b"test body",
+            status=202,
+        )
         self.crawler.stats.open_spider(self.spider)
 
     def tearDown(self):
-        self.crawler.stats.close_spider(self.spider, '')
+        self.crawler.stats.close_spider(self.spider, "")
         shutil.rmtree(self.tmpdir)
 
     def _get_settings(self, **new_settings):
         settings = {
-            'HTTPCACHE_ENABLED': True,
-            'HTTPCACHE_DIR': self.tmpdir,
-            'HTTPCACHE_EXPIRATION_SECS': 1,
-            'HTTPCACHE_IGNORE_HTTP_CODES': [],
-            'HTTPCACHE_POLICY': self.policy_class,
-            'HTTPCACHE_STORAGE': self.storage_class,
+            "HTTPCACHE_ENABLED": True,
+            "HTTPCACHE_DIR": self.tmpdir,
+            "HTTPCACHE_EXPIRATION_SECS": 1,
+            "HTTPCACHE_IGNORE_HTTP_CODES": [],
+            "HTTPCACHE_POLICY": self.policy_class,
+            "HTTPCACHE_STORAGE": self.storage_class,
         }
         settings.update(new_settings)
         return Settings(settings)
@@ -82,26 +83,32 @@ class _BaseTest(unittest.TestCase):
 
     def assertEqualRequestButWithCacheValidators(self, request1, request2):
         self.assertEqual(request1.url, request2.url)
-        assert b'If-None-Match' not in request1.headers
-        assert b'If-Modified-Since' not in request1.headers
-        assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
+        assert b"If-None-Match" not in request1.headers
+        assert b"If-Modified-Since" not in request1.headers
+        assert any(
+            h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since")
+        )
         self.assertEqual(request1.body, request2.body)
 
     def test_dont_cache(self):
         with self._middleware() as mw:
-            self.request.meta['dont_cache'] = True
+            self.request.meta["dont_cache"] = True
             mw.process_response(self.request, self.response, self.spider)
-            self.assertEqual(mw.storage.retrieve_response(self.spider, self.request), None)
+            self.assertEqual(
+                mw.storage.retrieve_response(self.spider, self.request), None
+            )
 
         with self._middleware() as mw:
-            self.request.meta['dont_cache'] = False
+            self.request.meta["dont_cache"] = False
             mw.process_response(self.request, self.response, self.spider)
             if mw.policy.should_cache_response(self.response, self.request):
-                self.assertIsInstance(mw.storage.retrieve_response(self.spider, self.request), self.response.__class__)
+                self.assertIsInstance(
+                    mw.storage.retrieve_response(self.spider, self.request),
+                    self.response.__class__,
+                )
 
 
 class DefaultStorageTest(_BaseTest):
-
     def test_storage(self):
         with self._storage() as storage:
             request2 = self.request.copy()
@@ -128,8 +135,8 @@ class DefaultStorageTest(_BaseTest):
         with self._storage() as storage:
             assert storage.retrieve_response(self.spider, self.request) is None
             response = Response(
-                'http://www.example.com',
-                body=b'<!DOCTYPE html>\n<title>.</title>',
+                "http://www.example.com",
+                body=b"<!DOCTYPE html>\n<title>.</title>",
                 status=202,
             )
             storage.store_response(self.spider, self.request, response)
@@ -140,15 +147,15 @@ class DefaultStorageTest(_BaseTest):
 
 class DbmStorageTest(DefaultStorageTest):
 
-    storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage'
+    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
 
 
 class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
-    dbm_module = 'tests.mocks.dummydbm'
+    dbm_module = "tests.mocks.dummydbm"
 
     def _get_settings(self, **new_settings):
-        new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
+        new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module)
         return super()._get_settings(**new_settings)
 
     def test_custom_dbm_module_loaded(self):
@@ -159,19 +166,18 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
 class FilesystemStorageTest(DefaultStorageTest):
 
-    storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+    storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
 
 class FilesystemStorageGzipTest(FilesystemStorageTest):
-
     def _get_settings(self, **new_settings):
-        new_settings.setdefault('HTTPCACHE_GZIP', True)
+        new_settings.setdefault("HTTPCACHE_GZIP", True)
         return super()._get_settings(**new_settings)
 
 
 class DummyPolicyTest(_BaseTest):
 
-    policy_class = 'scrapy.extensions.httpcache.DummyPolicy'
+    policy_class = "scrapy.extensions.httpcache.DummyPolicy"
 
     def test_middleware(self):
         with self._middleware() as mw:
@@ -180,31 +186,33 @@ class DummyPolicyTest(_BaseTest):
             response = mw.process_request(self.request, self.spider)
             assert isinstance(response, HtmlResponse)
             self.assertEqualResponse(self.response, response)
-            assert 'cached' in response.flags
+            assert "cached" in response.flags
 
     def test_different_request_response_urls(self):
         with self._middleware() as mw:
-            req = Request('http://host.com/path')
-            res = Response('http://host2.net/test.html')
+            req = Request("http://host.com/path")
+            res = Response("http://host2.net/test.html")
             assert mw.process_request(req, self.spider) is None
             mw.process_response(req, res, self.spider)
             cached = mw.process_request(req, self.spider)
             assert isinstance(cached, Response)
             self.assertEqualResponse(res, cached)
-            assert 'cached' in cached.flags
+            assert "cached" in cached.flags
 
     def test_middleware_ignore_missing(self):
         with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
-            self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider)
+            self.assertRaises(
+                IgnoreRequest, mw.process_request, self.request, self.spider
+            )
             mw.process_response(self.request, self.response, self.spider)
             response = mw.process_request(self.request, self.spider)
             assert isinstance(response, HtmlResponse)
             self.assertEqualResponse(self.response, response)
-            assert 'cached' in response.flags
+            assert "cached" in response.flags
 
     def test_middleware_ignore_schemes(self):
         # http responses are cached by default
-        req, res = Request('http://test.com/'), Response('http://test.com/')
+        req, res = Request("http://test.com/"), Response("http://test.com/")
         with self._middleware() as mw:
             assert mw.process_request(req, self.spider) is None
             mw.process_response(req, res, self.spider)
@@ -212,10 +220,10 @@ class DummyPolicyTest(_BaseTest):
             cached = mw.process_request(req, self.spider)
             assert isinstance(cached, Response), type(cached)
             self.assertEqualResponse(res, cached)
-            assert 'cached' in cached.flags
+            assert "cached" in cached.flags
 
         # file response is not cached by default
-        req, res = Request('file:///tmp/t.txt'), Response('file:///tmp/t.txt')
+        req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt")
         with self._middleware() as mw:
             assert mw.process_request(req, self.spider) is None
             mw.process_response(req, res, self.spider)
@@ -224,7 +232,7 @@ class DummyPolicyTest(_BaseTest):
             assert mw.process_request(req, self.spider) is None
 
         # s3 scheme response is cached by default
-        req, res = Request('s3://bucket/key'), Response('http://bucket/key')
+        req, res = Request("s3://bucket/key"), Response("http://bucket/key")
         with self._middleware() as mw:
             assert mw.process_request(req, self.spider) is None
             mw.process_response(req, res, self.spider)
@@ -232,11 +240,11 @@ class DummyPolicyTest(_BaseTest):
             cached = mw.process_request(req, self.spider)
             assert isinstance(cached, Response), type(cached)
             self.assertEqualResponse(res, cached)
-            assert 'cached' in cached.flags
+            assert "cached" in cached.flags
 
         # ignore s3 scheme
-        req, res = Request('s3://bucket/key2'), Response('http://bucket/key2')
-        with self._middleware(HTTPCACHE_IGNORE_SCHEMES=['s3']) as mw:
+        req, res = Request("s3://bucket/key2"), Response("http://bucket/key2")
+        with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:
             assert mw.process_request(req, self.spider) is None
             mw.process_response(req, res, self.spider)
 
@@ -258,12 +266,12 @@ class DummyPolicyTest(_BaseTest):
             response = mw.process_request(self.request, self.spider)
             assert isinstance(response, HtmlResponse)
             self.assertEqualResponse(self.response, response)
-            assert 'cached' in response.flags
+            assert "cached" in response.flags
 
 
 class RFC2616PolicyTest(DefaultStorageTest):
 
-    policy_class = 'scrapy.extensions.httpcache.RFC2616Policy'
+    policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
     def _process_requestresponse(self, mw, request, response):
         result = None
@@ -276,17 +284,18 @@ class RFC2616PolicyTest(DefaultStorageTest):
             assert isinstance(result, Response)
             return result
         except Exception:
-            print('Request', request)
-            print('Response', response)
-            print('Result', result)
+            print("Request", request)
+            print("Response", response)
+            print("Result", result)
             raise
 
     def test_request_cacheability(self):
-        res0 = Response(self.request.url, status=200,
-                        headers={'Expires': self.tomorrow})
-        req0 = Request('http://example.com')
-        req1 = req0.replace(headers={'Cache-Control': 'no-store'})
-        req2 = req0.replace(headers={'Cache-Control': 'no-cache'})
+        res0 = Response(
+            self.request.url, status=200, headers={"Expires": self.tomorrow}
+        )
+        req0 = Request("http://example.com")
+        req1 = req0.replace(headers={"Cache-Control": "no-store"})
+        req2 = req0.replace(headers={"Cache-Control": "no-cache"})
         with self._middleware() as mw:
             # response for a request with no-store must not be cached
             res1 = self._process_requestresponse(mw, req1, res0)
@@ -294,227 +303,276 @@ class RFC2616PolicyTest(DefaultStorageTest):
             assert mw.storage.retrieve_response(self.spider, req1) is None
             # Re-do request without no-store and expect it to be cached
             res2 = self._process_requestresponse(mw, req0, res0)
-            assert 'cached' not in res2.flags
+            assert "cached" not in res2.flags
             res3 = mw.process_request(req0, self.spider)
-            assert 'cached' in res3.flags
+            assert "cached" in res3.flags
             self.assertEqualResponse(res2, res3)
             # request with no-cache directive must not return cached response
             # but it allows new response to be stored
-            res0b = res0.replace(body=b'foo')
+            res0b = res0.replace(body=b"foo")
             res4 = self._process_requestresponse(mw, req2, res0b)
             self.assertEqualResponse(res4, res0b)
-            assert 'cached' not in res4.flags
+            assert "cached" not in res4.flags
             res5 = self._process_requestresponse(mw, req0, None)
             self.assertEqualResponse(res5, res0b)
-            assert 'cached' in res5.flags
+            assert "cached" in res5.flags
 
     def test_response_cacheability(self):
         responses = [
             # 304 is not cacheable no matter what servers sends
             (False, 304, {}),
-            (False, 304, {'Last-Modified': self.yesterday}),
-            (False, 304, {'Expires': self.tomorrow}),
-            (False, 304, {'Etag': 'bar'}),
-            (False, 304, {'Cache-Control': 'max-age=3600'}),
+            (False, 304, {"Last-Modified": self.yesterday}),
+            (False, 304, {"Expires": self.tomorrow}),
+            (False, 304, {"Etag": "bar"}),
+            (False, 304, {"Cache-Control": "max-age=3600"}),
             # Always obey no-store cache control
-            (False, 200, {'Cache-Control': 'no-store'}),
-            (False, 200, {'Cache-Control': 'no-store, max-age=300'}),  # invalid
-            (False, 200, {'Cache-Control': 'no-store', 'Expires': self.tomorrow}),  # invalid
+            (False, 200, {"Cache-Control": "no-store"}),
+            (False, 200, {"Cache-Control": "no-store, max-age=300"}),  # invalid
+            (
+                False,
+                200,
+                {"Cache-Control": "no-store", "Expires": self.tomorrow},
+            ),  # invalid
             # Ignore responses missing expiration and/or validation headers
             (False, 200, {}),
             (False, 302, {}),
             (False, 307, {}),
             (False, 404, {}),
             # Cache responses with expiration and/or validation headers
-            (True, 200, {'Last-Modified': self.yesterday}),
-            (True, 203, {'Last-Modified': self.yesterday}),
-            (True, 300, {'Last-Modified': self.yesterday}),
-            (True, 301, {'Last-Modified': self.yesterday}),
-            (True, 308, {'Last-Modified': self.yesterday}),
-            (True, 401, {'Last-Modified': self.yesterday}),
-            (True, 404, {'Cache-Control': 'public, max-age=600'}),
-            (True, 302, {'Expires': self.tomorrow}),
-            (True, 200, {'Etag': 'foo'}),
+            (True, 200, {"Last-Modified": self.yesterday}),
+            (True, 203, {"Last-Modified": self.yesterday}),
+            (True, 300, {"Last-Modified": self.yesterday}),
+            (True, 301, {"Last-Modified": self.yesterday}),
+            (True, 308, {"Last-Modified": self.yesterday}),
+            (True, 401, {"Last-Modified": self.yesterday}),
+            (True, 404, {"Cache-Control": "public, max-age=600"}),
+            (True, 302, {"Expires": self.tomorrow}),
+            (True, 200, {"Etag": "foo"}),
         ]
         with self._middleware() as mw:
             for idx, (shouldcache, status, headers) in enumerate(responses):
-                req0 = Request(f'http://example-{idx}.com')
+                req0 = Request(f"http://example-{idx}.com")
                 res0 = Response(req0.url, status=status, headers=headers)
                 res1 = self._process_requestresponse(mw, req0, res0)
                 res304 = res0.replace(status=304)
-                res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0)
+                res2 = self._process_requestresponse(
+                    mw, req0, res304 if shouldcache else res0
+                )
                 self.assertEqualResponse(res1, res0)
                 self.assertEqualResponse(res2, res0)
                 resc = mw.storage.retrieve_response(self.spider, req0)
                 if shouldcache:
                     self.assertEqualResponse(resc, res1)
-                    assert 'cached' in res2.flags and res2.status != 304
+                    assert "cached" in res2.flags and res2.status != 304
                 else:
                     self.assertFalse(resc)
-                    assert 'cached' not in res2.flags
+                    assert "cached" not in res2.flags
 
         # cache unconditionally unless response contains no-store or is a 304
         with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw:
             for idx, (_, status, headers) in enumerate(responses):
-                shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304
-                req0 = Request(f'http://example2-{idx}.com')
+                shouldcache = (
+                    "no-store" not in headers.get("Cache-Control", "") and status != 304
+                )
+                req0 = Request(f"http://example2-{idx}.com")
                 res0 = Response(req0.url, status=status, headers=headers)
                 res1 = self._process_requestresponse(mw, req0, res0)
                 res304 = res0.replace(status=304)
-                res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0)
+                res2 = self._process_requestresponse(
+                    mw, req0, res304 if shouldcache else res0
+                )
                 self.assertEqualResponse(res1, res0)
                 self.assertEqualResponse(res2, res0)
                 resc = mw.storage.retrieve_response(self.spider, req0)
                 if shouldcache:
                     self.assertEqualResponse(resc, res1)
-                    assert 'cached' in res2.flags and res2.status != 304
+                    assert "cached" in res2.flags and res2.status != 304
                 else:
                     self.assertFalse(resc)
-                    assert 'cached' not in res2.flags
+                    assert "cached" not in res2.flags
 
     def test_cached_and_fresh(self):
         sampledata = [
-            (200, {'Date': self.yesterday, 'Expires': self.tomorrow}),
-            (200, {'Date': self.yesterday, 'Cache-Control': 'max-age=86405'}),
-            (200, {'Age': '299', 'Cache-Control': 'max-age=300'}),
+            (200, {"Date": self.yesterday, "Expires": self.tomorrow}),
+            (200, {"Date": self.yesterday, "Cache-Control": "max-age=86405"}),
+            (200, {"Age": "299", "Cache-Control": "max-age=300"}),
             # Obey max-age if present over any others
-            (200, {'Date': self.today,
-                   'Age': '86405',
-                   'Cache-Control': 'max-age=' + str(86400 * 3),
-                   'Expires': self.yesterday,
-                   'Last-Modified': self.yesterday,
-                   }),
+            (
+                200,
+                {
+                    "Date": self.today,
+                    "Age": "86405",
+                    "Cache-Control": "max-age=" + str(86400 * 3),
+                    "Expires": self.yesterday,
+                    "Last-Modified": self.yesterday,
+                },
+            ),
             # obey Expires if max-age is not present
-            (200, {'Date': self.yesterday,
-                   'Age': '86400',
-                   'Cache-Control': 'public',
-                   'Expires': self.tomorrow,
-                   'Last-Modified': self.yesterday,
-                   }),
+            (
+                200,
+                {
+                    "Date": self.yesterday,
+                    "Age": "86400",
+                    "Cache-Control": "public",
+                    "Expires": self.tomorrow,
+                    "Last-Modified": self.yesterday,
+                },
+            ),
             # Default missing Date header to right now
-            (200, {'Expires': self.tomorrow}),
+            (200, {"Expires": self.tomorrow}),
             # Firefox - Expires if age is greater than 10% of (Date - Last-Modified)
-            (200, {'Date': self.today, 'Last-Modified': self.yesterday, 'Age': str(86400 / 10 - 1)}),
+            (
+                200,
+                {
+                    "Date": self.today,
+                    "Last-Modified": self.yesterday,
+                    "Age": str(86400 / 10 - 1),
+                },
+            ),
             # Firefox - Set one year maxage to permanent redirects missing expiration info
-            (300, {}), (301, {}), (308, {}),
+            (300, {}),
+            (301, {}),
+            (308, {}),
         ]
         with self._middleware() as mw:
             for idx, (status, headers) in enumerate(sampledata):
-                req0 = Request(f'http://example-{idx}.com')
+                req0 = Request(f"http://example-{idx}.com")
                 res0 = Response(req0.url, status=status, headers=headers)
                 # cache fresh response
                 res1 = self._process_requestresponse(mw, req0, res0)
                 self.assertEqualResponse(res1, res0)
-                assert 'cached' not in res1.flags
+                assert "cached" not in res1.flags
                 # return fresh cached response without network interaction
                 res2 = self._process_requestresponse(mw, req0, None)
                 self.assertEqualResponse(res1, res2)
-                assert 'cached' in res2.flags
+                assert "cached" in res2.flags
                 # validate cached response if request max-age set as 0
-                req1 = req0.replace(headers={'Cache-Control': 'max-age=0'})
+                req1 = req0.replace(headers={"Cache-Control": "max-age=0"})
                 res304 = res0.replace(status=304)
                 assert mw.process_request(req1, self.spider) is None
                 res3 = self._process_requestresponse(mw, req1, res304)
                 self.assertEqualResponse(res1, res3)
-                assert 'cached' in res3.flags
+                assert "cached" in res3.flags
 
     def test_cached_and_stale(self):
         sampledata = [
-            (200, {'Date': self.today, 'Expires': self.yesterday}),
-            (200, {'Date': self.today, 'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
-            (200, {'Expires': self.yesterday}),
-            (200, {'Expires': self.yesterday, 'ETag': 'foo'}),
-            (200, {'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
-            (200, {'Expires': self.tomorrow, 'Age': '86405'}),
-            (200, {'Cache-Control': 'max-age=86400', 'Age': '86405'}),
+            (200, {"Date": self.today, "Expires": self.yesterday}),
+            (
+                200,
+                {
+                    "Date": self.today,
+                    "Expires": self.yesterday,
+                    "Last-Modified": self.yesterday,
+                },
+            ),
+            (200, {"Expires": self.yesterday}),
+            (200, {"Expires": self.yesterday, "ETag": "foo"}),
+            (200, {"Expires": self.yesterday, "Last-Modified": self.yesterday}),
+            (200, {"Expires": self.tomorrow, "Age": "86405"}),
+            (200, {"Cache-Control": "max-age=86400", "Age": "86405"}),
             # no-cache forces expiration, also revalidation if validators exists
-            (200, {'Cache-Control': 'no-cache'}),
-            (200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}),
-            (200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}),
-            (200, {'Cache-Control': 'no-cache,must-revalidate', 'Last-Modified': self.yesterday}),
-            (200, {'Cache-Control': 'must-revalidate', 'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
-            (200, {'Cache-Control': 'max-age=86400,must-revalidate', 'Age': '86405'}),
+            (200, {"Cache-Control": "no-cache"}),
+            (200, {"Cache-Control": "no-cache", "ETag": "foo"}),
+            (200, {"Cache-Control": "no-cache", "Last-Modified": self.yesterday}),
+            (
+                200,
+                {
+                    "Cache-Control": "no-cache,must-revalidate",
+                    "Last-Modified": self.yesterday,
+                },
+            ),
+            (
+                200,
+                {
+                    "Cache-Control": "must-revalidate",
+                    "Expires": self.yesterday,
+                    "Last-Modified": self.yesterday,
+                },
+            ),
+            (200, {"Cache-Control": "max-age=86400,must-revalidate", "Age": "86405"}),
         ]
         with self._middleware() as mw:
             for idx, (status, headers) in enumerate(sampledata):
-                req0 = Request(f'http://example-{idx}.com')
+                req0 = Request(f"http://example-{idx}.com")
                 res0a = Response(req0.url, status=status, headers=headers)
                 # cache expired response
                 res1 = self._process_requestresponse(mw, req0, res0a)
                 self.assertEqualResponse(res1, res0a)
-                assert 'cached' not in res1.flags
+                assert "cached" not in res1.flags
                 # Same request but as cached response is stale a new response must
                 # be returned
-                res0b = res0a.replace(body=b'bar')
+                res0b = res0a.replace(body=b"bar")
                 res2 = self._process_requestresponse(mw, req0, res0b)
                 self.assertEqualResponse(res2, res0b)
-                assert 'cached' not in res2.flags
-                cc = headers.get('Cache-Control', '')
+                assert "cached" not in res2.flags
+                cc = headers.get("Cache-Control", "")
                 # Previous response expired too, subsequent request to same
                 # resource must revalidate and succeed on 304 if validators
                 # are present
-                if 'ETag' in headers or 'Last-Modified' in headers:
+                if "ETag" in headers or "Last-Modified" in headers:
                     res0c = res0b.replace(status=304)
                     res3 = self._process_requestresponse(mw, req0, res0c)
                     self.assertEqualResponse(res3, res0b)
-                    assert 'cached' in res3.flags
+                    assert "cached" in res3.flags
                     # get cached response on server errors unless must-revalidate
                     # in cached response
                     res0d = res0b.replace(status=500)
                     res4 = self._process_requestresponse(mw, req0, res0d)
-                    if 'must-revalidate' in cc:
-                        assert 'cached' not in res4.flags
+                    if "must-revalidate" in cc:
+                        assert "cached" not in res4.flags
                         self.assertEqualResponse(res4, res0d)
                     else:
-                        assert 'cached' in res4.flags
+                        assert "cached" in res4.flags
                         self.assertEqualResponse(res4, res0b)
                 # Requests with max-stale can fetch expired cached responses
                 # unless cached response has must-revalidate
-                req1 = req0.replace(headers={'Cache-Control': 'max-stale'})
+                req1 = req0.replace(headers={"Cache-Control": "max-stale"})
                 res5 = self._process_requestresponse(mw, req1, res0b)
                 self.assertEqualResponse(res5, res0b)
-                if 'no-cache' in cc or 'must-revalidate' in cc:
-                    assert 'cached' not in res5.flags
+                if "no-cache" in cc or "must-revalidate" in cc:
+                    assert "cached" not in res5.flags
                 else:
-                    assert 'cached' in res5.flags
+                    assert "cached" in res5.flags
 
     def test_process_exception(self):
         with self._middleware() as mw:
-            res0 = Response(self.request.url, headers={'Expires': self.yesterday})
+            res0 = Response(self.request.url, headers={"Expires": self.yesterday})
             req0 = Request(self.request.url)
             self._process_requestresponse(mw, req0, res0)
             for e in mw.DOWNLOAD_EXCEPTIONS:
                 # Simulate encountering an error on download attempts
                 assert mw.process_request(req0, self.spider) is None
-                res1 = mw.process_exception(req0, e('foo'), self.spider)
+                res1 = mw.process_exception(req0, e("foo"), self.spider)
                 # Use cached response as recovery
-                assert 'cached' in res1.flags
+                assert "cached" in res1.flags
                 self.assertEqualResponse(res0, res1)
             # Do not use cached response for unhandled exceptions
             mw.process_request(req0, self.spider)
-            assert mw.process_exception(req0, Exception('foo'), self.spider) is None
+            assert mw.process_exception(req0, Exception("foo"), self.spider) is None
 
     def test_ignore_response_cache_controls(self):
         sampledata = [
-            (200, {'Date': self.yesterday, 'Expires': self.tomorrow}),
-            (200, {'Date': self.yesterday, 'Cache-Control': 'no-store,max-age=86405'}),
-            (200, {'Age': '299', 'Cache-Control': 'max-age=300,no-cache'}),
-            (300, {'Cache-Control': 'no-cache'}),
-            (200, {'Expires': self.tomorrow, 'Cache-Control': 'no-store'}),
+            (200, {"Date": self.yesterday, "Expires": self.tomorrow}),
+            (200, {"Date": self.yesterday, "Cache-Control": "no-store,max-age=86405"}),
+            (200, {"Age": "299", "Cache-Control": "max-age=300,no-cache"}),
+            (300, {"Cache-Control": "no-cache"}),
+            (200, {"Expires": self.tomorrow, "Cache-Control": "no-store"}),
         ]
-        with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw:
+        with self._middleware(
+            HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=["no-cache", "no-store"]
+        ) as mw:
             for idx, (status, headers) in enumerate(sampledata):
-                req0 = Request(f'http://example-{idx}.com')
+                req0 = Request(f"http://example-{idx}.com")
                 res0 = Response(req0.url, status=status, headers=headers)
                 # cache fresh response
                 res1 = self._process_requestresponse(mw, req0, res0)
                 self.assertEqualResponse(res1, res0)
-                assert 'cached' not in res1.flags
+                assert "cached" not in res1.flags
                 # return fresh cached response without network interaction
                 res2 = self._process_requestresponse(mw, req0, None)
                 self.assertEqualResponse(res1, res2)
-                assert 'cached' in res2.flags
+                assert "cached" in res2.flags
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index b0272143d..efae7c4e0 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -7,35 +7,40 @@ from warnings import catch_warnings
 from w3lib.encoding import resolve_encoding
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
+from scrapy.downloadermiddlewares.httpcompression import (
+    HttpCompressionMiddleware,
+    ACCEPTED_ENCODINGS,
+)
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
 from scrapy.utils.test import get_crawler
 from tests import tests_datadir
 
-SAMPLEDIR = Path(tests_datadir, 'compressed')
+SAMPLEDIR = Path(tests_datadir, "compressed")
 
 FORMAT = {
-    'gzip': ('html-gzip.bin', 'gzip'),
-    'x-gzip': ('html-gzip.bin', 'gzip'),
-    'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
-    'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
-    'br': ('html-br.bin', 'br'),
+    "gzip": ("html-gzip.bin", "gzip"),
+    "x-gzip": ("html-gzip.bin", "gzip"),
+    "rawdeflate": ("html-rawdeflate.bin", "deflate"),
+    "zlibdeflate": ("html-zlibdeflate.bin", "deflate"),
+    "br": ("html-br.bin", "br"),
     # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
-    'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
+    "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"),
     # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
-    'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
+    "zstd-static-no-content-size": ("html-zstd-static-no-content-size.bin", "zstd"),
     # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
-    'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'),
+    "zstd-streaming-no-content-size": (
+        "html-zstd-streaming-no-content-size.bin",
+        "zstd",
+    ),
 }
 
 
 class HttpCompressionTest(TestCase):
-
     def setUp(self):
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('scrapytest.org')
+        self.spider = self.crawler._create_spider("scrapytest.org")
         self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
         self.crawler.stats.open_spider(self.spider)
 
@@ -48,90 +53,93 @@ class HttpCompressionTest(TestCase):
         body = (SAMPLEDIR / samplefile).read_bytes()
 
         headers = {
-            'Server': 'Yaws/1.49 Yet Another Web Server',
-            'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
-            'Content-Length': len(body),
-            'Content-Type': 'text/html',
-            'Content-Encoding': contentencoding,
+            "Server": "Yaws/1.49 Yet Another Web Server",
+            "Date": "Sun, 08 Mar 2009 00:41:03 GMT",
+            "Content-Length": len(body),
+            "Content-Type": "text/html",
+            "Content-Encoding": contentencoding,
         }
 
-        response = Response('http://scrapytest.org/', body=body, headers=headers)
-        response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
+        response = Response("http://scrapytest.org/", body=body, headers=headers)
+        response.request = Request(
+            "http://scrapytest.org", headers={"Accept-Encoding": "gzip, deflate"}
+        )
         return response
 
     def assertStatsEqual(self, key, value):
         self.assertEqual(
             self.crawler.stats.get_value(key, spider=self.spider),
             value,
-            str(self.crawler.stats.get_stats(self.spider))
+            str(self.crawler.stats.get_stats(self.spider)),
         )
 
     def test_setting_false_compression_enabled(self):
         self.assertRaises(
             NotConfigured,
             HttpCompressionMiddleware.from_crawler,
-            get_crawler(settings_dict={'COMPRESSION_ENABLED': False})
+            get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),
         )
 
     def test_setting_default_compression_enabled(self):
         self.assertIsInstance(
             HttpCompressionMiddleware.from_crawler(get_crawler()),
-            HttpCompressionMiddleware
+            HttpCompressionMiddleware,
         )
 
     def test_setting_true_compression_enabled(self):
         self.assertIsInstance(
             HttpCompressionMiddleware.from_crawler(
-                get_crawler(settings_dict={'COMPRESSION_ENABLED': True})
+                get_crawler(settings_dict={"COMPRESSION_ENABLED": True})
             ),
-            HttpCompressionMiddleware
+            HttpCompressionMiddleware,
         )
 
     def test_process_request(self):
-        request = Request('http://scrapytest.org')
-        assert 'Accept-Encoding' not in request.headers
+        request = Request("http://scrapytest.org")
+        assert "Accept-Encoding" not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'),
-                         b', '.join(ACCEPTED_ENCODINGS))
+        self.assertEqual(
+            request.headers.get("Accept-Encoding"), b", ".join(ACCEPTED_ENCODINGS)
+        )
 
     def test_process_response_gzip(self):
-        response = self._getresponse('gzip')
+        response = self._getresponse("gzip")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert "Content-Encoding" not in newresponse.headers
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzip_no_stats(self):
         mw = HttpCompressionMiddleware()
-        response = self._getresponse('gzip')
+        response = self._getresponse("gzip")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = mw.process_response(request, response, self.spider)
         self.assertEqual(mw.stats, None)
         assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert "Content-Encoding" not in newresponse.headers
 
     def test_process_response_br(self):
         try:
             import brotli  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
-        response = self._getresponse('br')
+        response = self._getresponse("br")
         request = response.request
-        self.assertEqual(response.headers['Content-Encoding'], b'br')
+        self.assertEqual(response.headers["Content-Encoding"], b"br")
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert 'Content-Encoding' not in newresponse.headers
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        assert "Content-Encoding" not in newresponse.headers
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_zstd(self):
         try:
@@ -140,11 +148,11 @@ class HttpCompressionTest(TestCase):
             raise SkipTest("no zstd support (zstandard)")
         raw_content = None
         for check_key in FORMAT:
-            if not check_key.startswith('zstd-'):
+            if not check_key.startswith("zstd-"):
                 continue
             response = self._getresponse(check_key)
             request = response.request
-            self.assertEqual(response.headers['Content-Encoding'], b'zstd')
+            self.assertEqual(response.headers["Content-Encoding"], b"zstd")
             newresponse = self.mw.process_response(request, response, self.spider)
             if raw_content is None:
                 raw_content = newresponse.body
@@ -152,145 +160,159 @@ class HttpCompressionTest(TestCase):
                 assert raw_content == newresponse.body
             assert newresponse is not response
             assert newresponse.body.startswith(b"<!DOCTYPE")
-            assert 'Content-Encoding' not in newresponse.headers
+            assert "Content-Encoding" not in newresponse.headers
 
     def test_process_response_rawdeflate(self):
-        response = self._getresponse('rawdeflate')
+        response = self._getresponse("rawdeflate")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'deflate')
+        self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74840)
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert "Content-Encoding" not in newresponse.headers
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74840)
 
     def test_process_response_zlibdelate(self):
-        response = self._getresponse('zlibdeflate')
+        response = self._getresponse("zlibdeflate")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'deflate')
+        self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74840)
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert "Content-Encoding" not in newresponse.headers
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74840)
 
     def test_process_response_plain(self):
-        response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
-        request = Request('http://scrapytest.org')
+        response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
+        request = Request("http://scrapytest.org")
 
-        assert not response.headers.get('Content-Encoding')
+        assert not response.headers.get("Content-Encoding")
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        self.assertStatsEqual('httpcompression/response_count', None)
-        self.assertStatsEqual('httpcompression/response_bytes', None)
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        self.assertStatsEqual("httpcompression/response_count", None)
+        self.assertStatsEqual("httpcompression/response_bytes", None)
 
     def test_multipleencodings(self):
-        response = self._getresponse('gzip')
-        response.headers['Content-Encoding'] = ['uuencode', 'gzip']
+        response = self._getresponse("gzip")
+        response.headers["Content-Encoding"] = ["uuencode", "gzip"]
         request = response.request
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
+        self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"])
 
     def test_process_response_encoding_inside_body(self):
         headers = {
-            'Content-Type': 'text/html',
-            'Content-Encoding': 'gzip',
+            "Content-Type": "text/html",
+            "Content-Encoding": "gzip",
         }
         f = BytesIO()
-        plainbody = (b'<html><head><title>Some page</title>'
-                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
-        zf = GzipFile(fileobj=f, mode='wb')
+        plainbody = (
+            b"<html><head><title>Some page</title>"
+            b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
+        )
+        zf = GzipFile(fileobj=f, mode="wb")
         zf.write(plainbody)
         zf.close()
-        response = Response("http;//www.example.com/", headers=headers, body=f.getvalue())
+        response = Response(
+            "http;//www.example.com/", headers=headers, body=f.getvalue()
+        )
         request = Request("http://www.example.com/")
 
         newresponse = self.mw.process_response(request, response, self.spider)
         assert isinstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
-        self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 104)
+        self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 104)
 
     def test_process_response_force_recalculate_encoding(self):
         headers = {
-            'Content-Type': 'text/html',
-            'Content-Encoding': 'gzip',
+            "Content-Type": "text/html",
+            "Content-Encoding": "gzip",
         }
         f = BytesIO()
-        plainbody = (b'<html><head><title>Some page</title>'
-                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
-        zf = GzipFile(fileobj=f, mode='wb')
+        plainbody = (
+            b"<html><head><title>Some page</title>"
+            b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
+        )
+        zf = GzipFile(fileobj=f, mode="wb")
         zf.write(plainbody)
         zf.close()
-        response = HtmlResponse("http;//www.example.com/page.html", headers=headers, body=f.getvalue())
+        response = HtmlResponse(
+            "http;//www.example.com/page.html", headers=headers, body=f.getvalue()
+        )
         request = Request("http://www.example.com/")
 
         newresponse = self.mw.process_response(request, response, self.spider)
         assert isinstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
-        self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 104)
+        self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 104)
 
     def test_process_response_no_content_type_header(self):
         headers = {
-            'Content-Encoding': 'identity',
+            "Content-Encoding": "identity",
         }
-        plainbody = (b'<html><head><title>Some page</title>'
-                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
-        respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody)
-        response = respcls("http://www.example.com/index", headers=headers, body=plainbody)
+        plainbody = (
+            b"<html><head><title>Some page</title>"
+            b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
+        )
+        respcls = responsetypes.from_args(
+            url="http://www.example.com/index", headers=headers, body=plainbody
+        )
+        response = respcls(
+            "http://www.example.com/index", headers=headers, body=plainbody
+        )
         request = Request("http://www.example.com/index")
 
         newresponse = self.mw.process_response(request, response, self.spider)
         assert isinstance(newresponse, respcls)
         self.assertEqual(newresponse.body, plainbody)
-        self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 104)
+        self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 104)
 
     def test_process_response_gzipped_contenttype(self):
-        response = self._getresponse('gzip')
-        response.headers['Content-Type'] = 'application/gzip'
+        response = self._getresponse("gzip")
+        response.headers["Content-Type"] = "application/gzip"
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
-        response = self._getresponse('gzip')
-        response.headers['Content-Type'] = 'application/octet-stream'
+        response = self._getresponse("gzip")
+        response.headers["Content-Type"] = "application/octet-stream"
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
-        response = self._getresponse('x-gzip')
-        response.headers['Content-Type'] = 'binary/octet-stream'
+        response = self._getresponse("x-gzip")
+        response.headers["Content-Type"] = "binary/octet-stream"
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -298,8 +320,8 @@ class HttpCompressionTest(TestCase):
         to upper layers.
         """
         headers = {
-            'Content-Type': 'application/gzip',
-            'Content-Encoding': 'gzip',
+            "Content-Type": "application/gzip",
+            "Content-Encoding": "gzip",
         }
         # build a gzipped file (here, a sitemap)
         f = BytesIO()
@@ -318,42 +340,42 @@ class HttpCompressionTest(TestCase):
     <priority>0.8</priority>
   </url>
 </urlset>"""
-        gz_file = GzipFile(fileobj=f, mode='wb')
+        gz_file = GzipFile(fileobj=f, mode="wb")
         gz_file.write(plainbody)
         gz_file.close()
 
         # build a gzipped response body containing this gzipped file
         r = BytesIO()
-        gz_resp = GzipFile(fileobj=r, mode='wb')
+        gz_resp = GzipFile(fileobj=r, mode="wb")
         gz_resp.write(f.getvalue())
         gz_resp.close()
 
-        response = Response("http;//www.example.com/", headers=headers, body=r.getvalue())
+        response = Response(
+            "http;//www.example.com/", headers=headers, body=r.getvalue()
+        )
         request = Request("http://www.example.com/")
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertEqual(gunzip(newresponse.body), plainbody)
-        self.assertStatsEqual('httpcompression/response_count', 1)
-        self.assertStatsEqual('httpcompression/response_bytes', 230)
+        self.assertStatsEqual("httpcompression/response_count", 1)
+        self.assertStatsEqual("httpcompression/response_bytes", 230)
 
     def test_process_response_head_request_no_decode_required(self):
-        response = self._getresponse('gzip')
-        response.headers['Content-Type'] = 'application/gzip'
+        response = self._getresponse("gzip")
+        response.headers["Content-Type"] = "application/gzip"
         request = response.request
-        request.method = 'HEAD'
+        request.method = "HEAD"
         response = response.replace(body=None)
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
-        self.assertEqual(response.body, b'')
-        self.assertStatsEqual('httpcompression/response_count', None)
-        self.assertStatsEqual('httpcompression/response_bytes', None)
+        self.assertEqual(response.body, b"")
+        self.assertStatsEqual("httpcompression/response_count", None)
+        self.assertStatsEqual("httpcompression/response_bytes", None)
 
 
 class HttpCompressionSubclassTest(TestCase):
-
     def test_init_missing_stats(self):
         class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
-
             def __init__(self):
                 super().__init__()
 
@@ -361,7 +383,8 @@ class HttpCompressionSubclassTest(TestCase):
         with catch_warnings(record=True) as caught_warnings:
             HttpCompressionMiddlewareSubclass.from_crawler(crawler)
         messages = tuple(
-            str(warning.message) for warning in caught_warnings
+            str(warning.message)
+            for warning in caught_warnings
             if warning.category is ScrapyDeprecationWarning
         )
         self.assertEqual(
@@ -372,5 +395,5 @@ class HttpCompressionSubclassTest(TestCase):
                     "their '__init__' method to support a 'stats' parameter "
                     "or reimplement the 'from_crawler' method."
                 ),
-            )
+            ),
         )
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 44434f90e..ca125ba36 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -9,7 +9,7 @@ from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
-spider = Spider('foo')
+spider = Spider("foo")
 
 
 class TestHttpProxyMiddleware(TestCase):
@@ -23,184 +23,197 @@ class TestHttpProxyMiddleware(TestCase):
         os.environ = self._oldenv
 
     def test_not_enabled(self):
-        crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})
+        crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
         with pytest.raises(NotConfigured):
             HttpProxyMiddleware.from_crawler(crawler)
 
     def test_no_environment_proxies(self):
-        os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
+        os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"}
         mw = HttpProxyMiddleware()
 
-        for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
+        for url in ("http://e.com", "https://e.com", "file:///tmp/a"):
             req = Request(url)
             assert mw.process_request(req, spider) is None
             self.assertEqual(req.url, url)
             self.assertEqual(req.meta, {})
 
     def test_environment_proxies(self):
-        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
-        os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
-        os.environ.pop('file_proxy', None)
+        os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128"
+        os.environ["https_proxy"] = https_proxy = "http://proxy.for.https:8080"
+        os.environ.pop("file_proxy", None)
         mw = HttpProxyMiddleware()
 
         for url, proxy in [
-            ('http://e.com', http_proxy),
-            ('https://e.com', https_proxy),
-            ('file://tmp/a', None),
+            ("http://e.com", http_proxy),
+            ("https://e.com", https_proxy),
+            ("file://tmp/a", None),
         ]:
             req = Request(url)
             assert mw.process_request(req, spider) is None
             self.assertEqual(req.url, url)
-            self.assertEqual(req.meta.get('proxy'), proxy)
+            self.assertEqual(req.meta.get("proxy"), proxy)
 
     def test_proxy_precedence_meta(self):
-        os.environ['http_proxy'] = 'https://proxy.com'
+        os.environ["http_proxy"] = "https://proxy.com"
         mw = HttpProxyMiddleware()
-        req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'})
+        req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://new.proxy:3128'})
+        self.assertEqual(req.meta, {"proxy": "https://new.proxy:3128"})
 
     def test_proxy_auth(self):
-        os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
+        os.environ["http_proxy"] = "https://user:pass@proxy:3128"
         mw = HttpProxyMiddleware()
-        req = Request('http://scrapytest.org')
+        req = Request("http://scrapytest.org")
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcjpwYXNz")
         # proxy from request.meta
-        req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
+        req = Request(
+            "http://scrapytest.org",
+            meta={"proxy": "https://username:password@proxy:3128"},
+        )
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(
+            req.headers.get("Proxy-Authorization"), b"Basic dXNlcm5hbWU6cGFzc3dvcmQ="
+        )
 
     def test_proxy_auth_empty_passwd(self):
-        os.environ['http_proxy'] = 'https://user:@proxy:3128'
+        os.environ["http_proxy"] = "https://user:@proxy:3128"
         mw = HttpProxyMiddleware()
-        req = Request('http://scrapytest.org')
+        req = Request("http://scrapytest.org")
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcjo=")
         # proxy from request.meta
-        req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
+        req = Request(
+            "http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"}
+        )
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcm5hbWU6")
 
     def test_proxy_auth_encoding(self):
         # utf-8 encoding
-        os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
-        mw = HttpProxyMiddleware(auth_encoding='utf-8')
-        req = Request('http://scrapytest.org')
+        os.environ["http_proxy"] = "https://m\u00E1n:pass@proxy:3128"
+        mw = HttpProxyMiddleware(auth_encoding="utf-8")
+        req = Request("http://scrapytest.org")
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic bcOhbjpwYXNz")
 
         # proxy from request.meta
-        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
+        req = Request(
+            "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"}
+        )
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(
+            req.headers.get("Proxy-Authorization"), b"Basic w7xzZXI6cGFzcw=="
+        )
 
         # default latin-1 encoding
-        mw = HttpProxyMiddleware(auth_encoding='latin-1')
-        req = Request('http://scrapytest.org')
+        mw = HttpProxyMiddleware(auth_encoding="latin-1")
+        req = Request("http://scrapytest.org")
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic beFuOnBhc3M=")
 
         # proxy from request.meta, latin-1 encoding
-        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
+        req = Request(
+            "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"}
+        )
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
-        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
+        self.assertEqual(req.meta["proxy"], "https://proxy:3128")
+        self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic /HNlcjpwYXNz")
 
     def test_proxy_already_seted(self):
-        os.environ['http_proxy'] = 'https://proxy.for.http:3128'
+        os.environ["http_proxy"] = "https://proxy.for.http:3128"
         mw = HttpProxyMiddleware()
-        req = Request('http://noproxy.com', meta={'proxy': None})
+        req = Request("http://noproxy.com", meta={"proxy": None})
         assert mw.process_request(req, spider) is None
-        assert 'proxy' in req.meta and req.meta['proxy'] is None
+        assert "proxy" in req.meta and req.meta["proxy"] is None
 
     def test_no_proxy(self):
-        os.environ['http_proxy'] = 'https://proxy.for.http:3128'
+        os.environ["http_proxy"] = "https://proxy.for.http:3128"
         mw = HttpProxyMiddleware()
 
-        os.environ['no_proxy'] = '*'
-        req = Request('http://noproxy.com')
+        os.environ["no_proxy"] = "*"
+        req = Request("http://noproxy.com")
         assert mw.process_request(req, spider) is None
-        assert 'proxy' not in req.meta
+        assert "proxy" not in req.meta
 
-        os.environ['no_proxy'] = 'other.com'
-        req = Request('http://noproxy.com')
+        os.environ["no_proxy"] = "other.com"
+        req = Request("http://noproxy.com")
         assert mw.process_request(req, spider) is None
-        assert 'proxy' in req.meta
+        assert "proxy" in req.meta
 
-        os.environ['no_proxy'] = 'other.com,noproxy.com'
-        req = Request('http://noproxy.com')
+        os.environ["no_proxy"] = "other.com,noproxy.com"
+        req = Request("http://noproxy.com")
         assert mw.process_request(req, spider) is None
-        assert 'proxy' not in req.meta
+        assert "proxy" not in req.meta
 
         # proxy from meta['proxy'] takes precedence
-        os.environ['no_proxy'] = '*'
-        req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
+        os.environ["no_proxy"] = "*"
+        req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
+        self.assertEqual(req.meta, {"proxy": "http://proxy.com"})
 
     def test_no_proxy_invalid_values(self):
-        os.environ['no_proxy'] = '/var/run/docker.sock'
+        os.environ["no_proxy"] = "/var/run/docker.sock"
         mw = HttpProxyMiddleware()
         # '/var/run/docker.sock' may be used by the user for
         # no_proxy value but is not parseable and should be skipped
-        assert 'no' not in mw.proxies
+        assert "no" not in mw.proxies
 
     def test_add_proxy_without_credentials(self):
         middleware = HttpProxyMiddleware()
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = 'https://example.com'
+        request.meta["proxy"] = "https://example.com"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.meta["proxy"], "https://example.com")
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_add_proxy_with_credentials(self):
         middleware = HttpProxyMiddleware()
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = 'https://user1:password1@example.com'
+        request.meta["proxy"] = "https://user1:password1@example.com"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
         encoded_credentials = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
     def test_remove_proxy_without_credentials(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://example.com'},
+            "https://example.com",
+            meta={"proxy": "https://example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = None
+        request.meta["proxy"] = None
         assert middleware.process_request(request, spider) is None
-        self.assertIsNone(request.meta['proxy'])
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertIsNone(request.meta["proxy"])
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_remove_proxy_with_credentials(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = None
+        request.meta["proxy"] = None
         assert middleware.process_request(request, spider) is None
-        self.assertIsNone(request.meta['proxy'])
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertIsNone(request.meta["proxy"])
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_add_credentials(self):
         """If the proxy request meta switches to a proxy URL with the same
@@ -208,21 +221,21 @@ class TestHttpProxyMiddleware(TestCase):
         credentials must be used."""
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://example.com'},
+            "https://example.com",
+            meta={"proxy": "https://example.com"},
         )
         assert middleware.process_request(request, spider) is None
 
-        request.meta['proxy'] = 'https://user1:password1@example.com'
+        request.meta["proxy"] = "https://user1:password1@example.com"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
         encoded_credentials = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
     def test_change_credentials(self):
@@ -230,20 +243,20 @@ class TestHttpProxyMiddleware(TestCase):
         credentials, those new credentials must be used."""
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = 'https://user2:password2@example.com'
+        request.meta["proxy"] = "https://user2:password2@example.com"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
         encoded_credentials = middleware._basic_auth_header(
-            'user2',
-            'password2',
+            "user2",
+            "password2",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
     def test_remove_credentials(self):
@@ -256,93 +269,93 @@ class TestHttpProxyMiddleware(TestCase):
         """
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
 
-        request.meta['proxy'] = 'https://example.com'
+        request.meta["proxy"] = "https://example.com"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
         encoded_credentials = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
-        request.meta['proxy'] = 'https://example.com'
-        del request.headers[b'Proxy-Authorization']
+        request.meta["proxy"] = "https://example.com"
+        del request.headers[b"Proxy-Authorization"]
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.meta["proxy"], "https://example.com")
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_change_proxy_add_credentials(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://example.com'},
+            "https://example.com",
+            meta={"proxy": "https://example.com"},
         )
         assert middleware.process_request(request, spider) is None
 
-        request.meta['proxy'] = 'https://user1:password1@example.org'
+        request.meta["proxy"] = "https://user1:password1@example.org"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        self.assertEqual(request.meta["proxy"], "https://example.org")
         encoded_credentials = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
     def test_change_proxy_keep_credentials(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
 
-        request.meta['proxy'] = 'https://user1:password1@example.org'
+        request.meta["proxy"] = "https://user1:password1@example.org"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        self.assertEqual(request.meta["proxy"], "https://example.org")
         encoded_credentials = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
         # Make sure, indirectly, that _auth_proxy is updated.
-        request.meta['proxy'] = 'https://example.com'
+        request.meta["proxy"] = "https://example.com"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.meta["proxy"], "https://example.com")
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_change_proxy_change_credentials(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
 
-        request.meta['proxy'] = 'https://user2:password2@example.org'
+        request.meta["proxy"] = "https://user2:password2@example.org"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        self.assertEqual(request.meta["proxy"], "https://example.org")
         encoded_credentials = middleware._basic_auth_header(
-            'user2',
-            'password2',
+            "user2",
+            "password2",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
     def test_change_proxy_remove_credentials(self):
@@ -350,14 +363,14 @@ class TestHttpProxyMiddleware(TestCase):
         proxy and no credentials, no credentials must be used."""
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = 'https://example.org'
+        request.meta["proxy"] = "https://example.org"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta, {'proxy': 'https://example.org'})
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.meta, {"proxy": "https://example.org"})
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_change_proxy_remove_credentials_preremoved_header(self):
         """Corner case of proxy switch with credentials removal where the
@@ -368,36 +381,36 @@ class TestHttpProxyMiddleware(TestCase):
         """
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        request.meta['proxy'] = 'https://example.org'
-        del request.headers[b'Proxy-Authorization']
+        request.meta["proxy"] = "https://example.org"
+        del request.headers[b"Proxy-Authorization"]
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta, {'proxy': 'https://example.org'})
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.meta, {"proxy": "https://example.org"})
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_proxy_authentication_header_undefined_proxy(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            headers={'Proxy-Authorization': 'Basic foo'},
+            "https://example.com",
+            headers={"Proxy-Authorization": "Basic foo"},
         )
         assert middleware.process_request(request, spider) is None
-        self.assertNotIn('proxy', request.meta)
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertNotIn("proxy", request.meta)
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_proxy_authentication_header_disabled_proxy(self):
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            headers={'Proxy-Authorization': 'Basic foo'},
-            meta={'proxy': None},
+            "https://example.com",
+            headers={"Proxy-Authorization": "Basic foo"},
+            meta={"proxy": None},
         )
         assert middleware.process_request(request, spider) is None
-        self.assertIsNone(request.meta['proxy'])
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertIsNone(request.meta["proxy"])
+        self.assertNotIn(b"Proxy-Authorization", request.headers)
 
     def test_proxy_authentication_header_proxy_without_credentials(self):
         """As long as the proxy URL in request metadata remains the same, the
@@ -405,59 +418,59 @@ class TestHttpProxyMiddleware(TestCase):
         changed."""
         middleware = HttpProxyMiddleware()
         request = Request(
-            'https://example.com',
-            headers={'Proxy-Authorization': 'Basic foo'},
-            meta={'proxy': 'https://example.com'},
+            "https://example.com",
+            headers={"Proxy-Authorization": "Basic foo"},
+            meta={"proxy": "https://example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
+        self.assertEqual(request.headers["Proxy-Authorization"], b"Basic foo")
 
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
+        self.assertEqual(request.headers["Proxy-Authorization"], b"Basic foo")
 
-        request.headers['Proxy-Authorization'] = b'Basic bar'
+        request.headers["Proxy-Authorization"] = b"Basic bar"
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
+        self.assertEqual(request.headers["Proxy-Authorization"], b"Basic bar")
 
     def test_proxy_authentication_header_proxy_with_same_credentials(self):
         middleware = HttpProxyMiddleware()
         encoded_credentials = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         request = Request(
-            'https://example.com',
-            headers={'Proxy-Authorization': b'Basic ' + encoded_credentials},
-            meta={'proxy': 'https://user1:password1@example.com'},
+            "https://example.com",
+            headers={"Proxy-Authorization": b"Basic " + encoded_credentials},
+            meta={"proxy": "https://user1:password1@example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials,
         )
 
     def test_proxy_authentication_header_proxy_with_different_credentials(self):
         middleware = HttpProxyMiddleware()
         encoded_credentials1 = middleware._basic_auth_header(
-            'user1',
-            'password1',
+            "user1",
+            "password1",
         )
         request = Request(
-            'https://example.com',
-            headers={'Proxy-Authorization': b'Basic ' + encoded_credentials1},
-            meta={'proxy': 'https://user2:password2@example.com'},
+            "https://example.com",
+            headers={"Proxy-Authorization": b"Basic " + encoded_credentials1},
+            meta={"proxy": "https://user2:password2@example.com"},
         )
         assert middleware.process_request(request, spider) is None
-        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.meta["proxy"], "https://example.com")
         encoded_credentials2 = middleware._basic_auth_header(
-            'user2',
-            'password2',
+            "user2",
+            "password2",
         )
         self.assertEqual(
-            request.headers['Proxy-Authorization'],
-            b'Basic ' + encoded_credentials2,
+            request.headers["Proxy-Authorization"],
+            b"Basic " + encoded_credentials2,
         )
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 816ac1440..e2ff9ec2b 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,6 +1,9 @@
 import unittest
 
-from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
+from scrapy.downloadermiddlewares.redirect import (
+    RedirectMiddleware,
+    MetaRefreshMiddleware,
+)
 from scrapy.spiders import Spider
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response, HtmlResponse
@@ -8,24 +11,25 @@ from scrapy.utils.test import get_crawler
 
 
 class RedirectMiddlewareTest(unittest.TestCase):
-
     def setUp(self):
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('foo')
+        self.spider = self.crawler._create_spider("foo")
         self.mw = RedirectMiddleware.from_crawler(self.crawler)
 
     def test_priority_adjust(self):
-        req = Request('http://a.com')
-        rsp = Response('http://a.com', headers={'Location': 'http://a.com/redirected'}, status=301)
+        req = Request("http://a.com")
+        rsp = Response(
+            "http://a.com", headers={"Location": "http://a.com/redirected"}, status=301
+        )
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority > req.priority
 
     def test_redirect_3xx_permanent(self):
         def _test(method, status=301):
-            url = f'http://www.example.com/{status}'
-            url2 = 'http://www.example.com/redirected'
+            url = f"http://www.example.com/{status}"
+            url2 = "http://www.example.com/redirected"
             req = Request(url, method=method)
-            rsp = Response(url, headers={'Location': url2}, status=status)
+            rsp = Response(url, headers={"Location": url2}, status=status)
 
             req2 = self.mw.process_response(req, rsp, self.spider)
             assert isinstance(req2, Request)
@@ -33,33 +37,33 @@ class RedirectMiddlewareTest(unittest.TestCase):
             self.assertEqual(req2.method, method)
 
             # response without Location header but with status code is 3XX should be ignored
-            del rsp.headers['Location']
+            del rsp.headers["Location"]
             assert self.mw.process_response(req, rsp, self.spider) is rsp
 
-        _test('GET')
-        _test('POST')
-        _test('HEAD')
+        _test("GET")
+        _test("POST")
+        _test("HEAD")
 
-        _test('GET', status=307)
-        _test('POST', status=307)
-        _test('HEAD', status=307)
+        _test("GET", status=307)
+        _test("POST", status=307)
+        _test("HEAD", status=307)
 
-        _test('GET', status=308)
-        _test('POST', status=308)
-        _test('HEAD', status=308)
+        _test("GET", status=308)
+        _test("POST", status=308)
+        _test("HEAD", status=308)
 
     def test_dont_redirect(self):
-        url = 'http://www.example.com/301'
-        url2 = 'http://www.example.com/redirected'
-        req = Request(url, meta={'dont_redirect': True})
-        rsp = Response(url, headers={'Location': url2}, status=301)
+        url = "http://www.example.com/301"
+        url2 = "http://www.example.com/redirected"
+        req = Request(url, meta={"dont_redirect": True})
+        rsp = Response(url, headers={"Location": url2}, status=301)
 
         r = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(r, Response)
         assert r is rsp
 
         # Test that it redirects when dont_redirect is False
-        req = Request(url, meta={'dont_redirect': False})
+        req = Request(url, meta={"dont_redirect": False})
         rsp = Response(url2, status=200)
 
         r = self.mw.process_response(req, rsp, self.spider)
@@ -67,247 +71,313 @@ class RedirectMiddlewareTest(unittest.TestCase):
         assert r is rsp
 
     def test_redirect_302(self):
-        url = 'http://www.example.com/302'
-        url2 = 'http://www.example.com/redirected2'
-        req = Request(url, method='POST', body='test',
-                      headers={'Content-Type': 'text/plain', 'Content-length': '4'})
-        rsp = Response(url, headers={'Location': url2}, status=302)
+        url = "http://www.example.com/302"
+        url2 = "http://www.example.com/redirected2"
+        req = Request(
+            url,
+            method="POST",
+            body="test",
+            headers={"Content-Type": "text/plain", "Content-length": "4"},
+        )
+        rsp = Response(url, headers={"Location": url2}, status=302)
 
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, url2)
-        self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        self.assertEqual(req2.method, "GET")
+        assert (
+            "Content-Type" not in req2.headers
+        ), "Content-Type header must not be present in redirected request"
+        assert (
+            "Content-Length" not in req2.headers
+        ), "Content-Length header must not be present in redirected request"
         assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
 
         # response without Location header but with status code is 3XX should be ignored
-        del rsp.headers['Location']
+        del rsp.headers["Location"]
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_redirect_302_head(self):
-        url = 'http://www.example.com/302'
-        url2 = 'http://www.example.com/redirected2'
-        req = Request(url, method='HEAD')
-        rsp = Response(url, headers={'Location': url2}, status=302)
+        url = "http://www.example.com/302"
+        url2 = "http://www.example.com/redirected2"
+        req = Request(url, method="HEAD")
+        rsp = Response(url, headers={"Location": url2}, status=302)
 
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, url2)
-        self.assertEqual(req2.method, 'HEAD')
+        self.assertEqual(req2.method, "HEAD")
 
         # response without Location header but with status code is 3XX should be ignored
-        del rsp.headers['Location']
+        del rsp.headers["Location"]
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_redirect_302_relative(self):
-        url = 'http://www.example.com/302'
-        url2 = '///i8n.example2.com/302'
-        url3 = 'http://i8n.example2.com/302'
-        req = Request(url, method='HEAD')
-        rsp = Response(url, headers={'Location': url2}, status=302)
+        url = "http://www.example.com/302"
+        url2 = "///i8n.example2.com/302"
+        url3 = "http://i8n.example2.com/302"
+        req = Request(url, method="HEAD")
+        rsp = Response(url, headers={"Location": url2}, status=302)
 
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, url3)
-        self.assertEqual(req2.method, 'HEAD')
+        self.assertEqual(req2.method, "HEAD")
 
         # response without Location header but with status code is 3XX should be ignored
-        del rsp.headers['Location']
+        del rsp.headers["Location"]
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
-        req = Request('http://scrapytest.org/302')
-        rsp = Response('http://scrapytest.org/302', headers={'Location': '/redirected'}, status=302)
+        req = Request("http://scrapytest.org/302")
+        rsp = Response(
+            "http://scrapytest.org/302", headers={"Location": "/redirected"}, status=302
+        )
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
-        assert 'redirect_times' in req.meta
-        self.assertEqual(req.meta['redirect_times'], 1)
-        self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider)
+        assert "redirect_times" in req.meta
+        self.assertEqual(req.meta["redirect_times"], 1)
+        self.assertRaises(
+            IgnoreRequest, self.mw.process_response, req, rsp, self.spider
+        )
 
     def test_ttl(self):
         self.mw.max_redirect_times = 100
-        req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1})
-        rsp = Response('http://www.scrapytest.org/302', headers={'Location': '/redirected'}, status=302)
+        req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1})
+        rsp = Response(
+            "http://www.scrapytest.org/302",
+            headers={"Location": "/redirected"},
+            status=302,
+        )
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
-        self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider)
+        self.assertRaises(
+            IgnoreRequest, self.mw.process_response, req, rsp, self.spider
+        )
 
     def test_redirect_urls(self):
-        req1 = Request('http://scrapytest.org/first')
-        rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected'}, status=302)
+        req1 = Request("http://scrapytest.org/first")
+        rsp1 = Response(
+            "http://scrapytest.org/first",
+            headers={"Location": "/redirected"},
+            status=302,
+        )
         req2 = self.mw.process_response(req1, rsp1, self.spider)
-        rsp2 = Response('http://scrapytest.org/redirected', headers={'Location': '/redirected2'}, status=302)
+        rsp2 = Response(
+            "http://scrapytest.org/redirected",
+            headers={"Location": "/redirected2"},
+            status=302,
+        )
         req3 = self.mw.process_response(req2, rsp2, self.spider)
 
-        self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
-        self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
-        self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
+        self.assertEqual(req2.url, "http://scrapytest.org/redirected")
+        self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"])
+        self.assertEqual(req3.url, "http://scrapytest.org/redirected2")
         self.assertEqual(
-            req3.meta['redirect_urls'],
-            ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
+            req3.meta["redirect_urls"],
+            ["http://scrapytest.org/first", "http://scrapytest.org/redirected"],
         )
 
     def test_redirect_reasons(self):
-        req1 = Request('http://scrapytest.org/first')
-        rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301)
+        req1 = Request("http://scrapytest.org/first")
+        rsp1 = Response(
+            "http://scrapytest.org/first",
+            headers={"Location": "/redirected1"},
+            status=301,
+        )
         req2 = self.mw.process_response(req1, rsp1, self.spider)
-        rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301)
+        rsp2 = Response(
+            "http://scrapytest.org/redirected1",
+            headers={"Location": "/redirected2"},
+            status=301,
+        )
         req3 = self.mw.process_response(req2, rsp2, self.spider)
 
-        self.assertEqual(req2.meta['redirect_reasons'], [301])
-        self.assertEqual(req3.meta['redirect_reasons'], [301, 301])
+        self.assertEqual(req2.meta["redirect_reasons"], [301])
+        self.assertEqual(req3.meta["redirect_reasons"], [301, 301])
 
     def test_spider_handling(self):
-        smartspider = self.crawler._create_spider('smarty')
+        smartspider = self.crawler._create_spider("smarty")
         smartspider.handle_httpstatus_list = [404, 301, 302]
-        url = 'http://www.example.com/301'
-        url2 = 'http://www.example.com/redirected'
+        url = "http://www.example.com/301"
+        url2 = "http://www.example.com/redirected"
         req = Request(url)
-        rsp = Response(url, headers={'Location': url2}, status=301)
+        rsp = Response(url, headers={"Location": url2}, status=301)
         r = self.mw.process_response(req, rsp, smartspider)
         self.assertIs(r, rsp)
 
     def test_request_meta_handling(self):
-        url = 'http://www.example.com/301'
-        url2 = 'http://www.example.com/redirected'
+        url = "http://www.example.com/301"
+        url2 = "http://www.example.com/redirected"
 
         def _test_passthrough(req):
-            rsp = Response(url, headers={'Location': url2}, status=301, request=req)
+            rsp = Response(url, headers={"Location": url2}, status=301, request=req)
             r = self.mw.process_response(req, rsp, self.spider)
             self.assertIs(r, rsp)
-        _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
-        _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
+
+        _test_passthrough(
+            Request(url, meta={"handle_httpstatus_list": [404, 301, 302]})
+        )
+        _test_passthrough(Request(url, meta={"handle_httpstatus_all": True}))
 
     def test_latin1_location(self):
-        req = Request('http://scrapytest.org/first')
-        latin1_location = '/ação'.encode('latin1')  # HTTP historically supports latin1
-        resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
+        req = Request("http://scrapytest.org/first")
+        latin1_location = "/ação".encode("latin1")  # HTTP historically supports latin1
+        resp = Response(
+            "http://scrapytest.org/first",
+            headers={"Location": latin1_location},
+            status=302,
+        )
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
+        perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o"
         self.assertEqual(perc_encoded_utf8_url, req_result.url)
 
     def test_utf8_location(self):
-        req = Request('http://scrapytest.org/first')
-        utf8_location = '/ação'.encode('utf-8')  # header using UTF-8 encoding
-        resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
+        req = Request("http://scrapytest.org/first")
+        utf8_location = "/ação".encode("utf-8")  # header using UTF-8 encoding
+        resp = Response(
+            "http://scrapytest.org/first",
+            headers={"Location": utf8_location},
+            status=302,
+        )
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
+        perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o"
         self.assertEqual(perc_encoded_utf8_url, req_result.url)
 
 
 class MetaRefreshMiddlewareTest(unittest.TestCase):
-
     def setUp(self):
         crawler = get_crawler(Spider)
-        self.spider = crawler._create_spider('foo')
+        self.spider = crawler._create_spider("foo")
         self.mw = MetaRefreshMiddleware.from_crawler(crawler)
 
-    def _body(self, interval=5, url='http://example.org/newpage'):
+    def _body(self, interval=5, url="http://example.org/newpage"):
         html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
-        return html.encode('utf-8')
+        return html.encode("utf-8")
 
     def test_priority_adjust(self):
-        req = Request('http://a.com')
+        req = Request("http://a.com")
         rsp = HtmlResponse(req.url, body=self._body())
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority > req.priority
 
     def test_meta_refresh(self):
-        req = Request(url='http://example.org')
+        req = Request(url="http://example.org")
         rsp = HtmlResponse(req.url, body=self._body())
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
-        self.assertEqual(req2.url, 'http://example.org/newpage')
+        self.assertEqual(req2.url, "http://example.org/newpage")
 
     def test_meta_refresh_with_high_interval(self):
         # meta-refresh with high intervals don't trigger redirects
-        req = Request(url='http://example.org')
-        rsp = HtmlResponse(url='http://example.org',
-                           body=self._body(interval=1000),
-                           encoding='utf-8')
+        req = Request(url="http://example.org")
+        rsp = HtmlResponse(
+            url="http://example.org", body=self._body(interval=1000), encoding="utf-8"
+        )
         rsp2 = self.mw.process_response(req, rsp, self.spider)
         assert rsp is rsp2
 
     def test_meta_refresh_trough_posted_request(self):
-        req = Request(url='http://example.org', method='POST', body='test',
-                      headers={'Content-Type': 'text/plain', 'Content-length': '4'})
+        req = Request(
+            url="http://example.org",
+            method="POST",
+            body="test",
+            headers={"Content-Type": "text/plain", "Content-length": "4"},
+        )
         rsp = HtmlResponse(req.url, body=self._body())
         req2 = self.mw.process_response(req, rsp, self.spider)
 
         assert isinstance(req2, Request)
-        self.assertEqual(req2.url, 'http://example.org/newpage')
-        self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        self.assertEqual(req2.url, "http://example.org/newpage")
+        self.assertEqual(req2.method, "GET")
+        assert (
+            "Content-Type" not in req2.headers
+        ), "Content-Type header must not be present in redirected request"
+        assert (
+            "Content-Length" not in req2.headers
+        ), "Content-Length header must not be present in redirected request"
         assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
 
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
-        req = Request('http://scrapytest.org/max')
+        req = Request("http://scrapytest.org/max")
         rsp = HtmlResponse(req.url, body=self._body())
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
-        assert 'redirect_times' in req.meta
-        self.assertEqual(req.meta['redirect_times'], 1)
-        self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider)
+        assert "redirect_times" in req.meta
+        self.assertEqual(req.meta["redirect_times"], 1)
+        self.assertRaises(
+            IgnoreRequest, self.mw.process_response, req, rsp, self.spider
+        )
 
     def test_ttl(self):
         self.mw.max_redirect_times = 100
-        req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1})
+        req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1})
         rsp = HtmlResponse(req.url, body=self._body())
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
-        self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider)
+        self.assertRaises(
+            IgnoreRequest, self.mw.process_response, req, rsp, self.spider
+        )
 
     def test_redirect_urls(self):
-        req1 = Request('http://scrapytest.org/first')
-        rsp1 = HtmlResponse(req1.url, body=self._body(url='/redirected'))
+        req1 = Request("http://scrapytest.org/first")
+        rsp1 = HtmlResponse(req1.url, body=self._body(url="/redirected"))
         req2 = self.mw.process_response(req1, rsp1, self.spider)
         assert isinstance(req2, Request), req2
-        rsp2 = HtmlResponse(req2.url, body=self._body(url='/redirected2'))
+        rsp2 = HtmlResponse(req2.url, body=self._body(url="/redirected2"))
         req3 = self.mw.process_response(req2, rsp2, self.spider)
         assert isinstance(req3, Request), req3
-        self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
-        self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
-        self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
+        self.assertEqual(req2.url, "http://scrapytest.org/redirected")
+        self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"])
+        self.assertEqual(req3.url, "http://scrapytest.org/redirected2")
         self.assertEqual(
-            req3.meta['redirect_urls'],
-            ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
+            req3.meta["redirect_urls"],
+            ["http://scrapytest.org/first", "http://scrapytest.org/redirected"],
         )
 
     def test_redirect_reasons(self):
-        req1 = Request('http://scrapytest.org/first')
-        rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected'))
+        req1 = Request("http://scrapytest.org/first")
+        rsp1 = HtmlResponse(
+            "http://scrapytest.org/first", body=self._body(url="/redirected")
+        )
         req2 = self.mw.process_response(req1, rsp1, self.spider)
-        rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1'))
+        rsp2 = HtmlResponse(
+            "http://scrapytest.org/redirected", body=self._body(url="/redirected1")
+        )
         req3 = self.mw.process_response(req2, rsp2, self.spider)
 
-        self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])
-        self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])
+        self.assertEqual(req2.meta["redirect_reasons"], ["meta refresh"])
+        self.assertEqual(
+            req3.meta["redirect_reasons"], ["meta refresh", "meta refresh"]
+        )
 
     def test_ignore_tags_default(self):
-        req = Request(url='http://example.org')
-        body = ('''<noscript><meta http-equiv="refresh" '''
-                '''content="0;URL='http://example.org/newpage'"></noscript>''')
+        req = Request(url="http://example.org")
+        body = (
+            """<noscript><meta http-equiv="refresh" """
+            """content="0;URL='http://example.org/newpage'"></noscript>"""
+        )
         rsp = HtmlResponse(req.url, body=body.encode())
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
-        self.assertEqual(req2.url, 'http://example.org/newpage')
+        self.assertEqual(req2.url, "http://example.org/newpage")
 
     def test_ignore_tags_1_x_list(self):
         """Test that Scrapy 1.x behavior remains possible"""
-        settings = {'METAREFRESH_IGNORE_TAGS': ['script', 'noscript']}
+        settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]}
         crawler = get_crawler(Spider, settings)
         mw = MetaRefreshMiddleware.from_crawler(crawler)
-        req = Request(url='http://example.org')
-        body = ('''<noscript><meta http-equiv="refresh" '''
-                '''content="0;URL='http://example.org/newpage'"></noscript>''')
+        req = Request(url="http://example.org")
+        body = (
+            """<noscript><meta http-equiv="refresh" """
+            """content="0;URL='http://example.org/newpage'"></noscript>"""
+        )
         rsp = HtmlResponse(req.url, body=body.encode())
         response = mw.process_response(req, rsp, self.spider)
         assert isinstance(response, Response)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 915bd3a3e..cadd647ad 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -22,65 +22,68 @@ from scrapy.utils.test import get_crawler
 class RetryTest(unittest.TestCase):
     def setUp(self):
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('foo')
+        self.spider = self.crawler._create_spider("foo")
         self.mw = RetryMiddleware.from_crawler(self.crawler)
         self.mw.max_retry_times = 2
 
     def test_priority_adjust(self):
-        req = Request('http://www.scrapytest.org/503')
-        rsp = Response('http://www.scrapytest.org/503', body=b'', status=503)
+        req = Request("http://www.scrapytest.org/503")
+        rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority < req.priority
 
     def test_404(self):
-        req = Request('http://www.scrapytest.org/404')
-        rsp = Response('http://www.scrapytest.org/404', body=b'', status=404)
+        req = Request("http://www.scrapytest.org/404")
+        rsp = Response("http://www.scrapytest.org/404", body=b"", status=404)
 
         # dont retry 404s
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_dont_retry(self):
-        req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True})
-        rsp = Response('http://www.scrapytest.org/503', body=b'', status=503)
+        req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
+        rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
 
         # first retry
         r = self.mw.process_response(req, rsp, self.spider)
         assert r is rsp
 
         # Test retry when dont_retry set to False
-        req = Request('http://www.scrapytest.org/503', meta={'dont_retry': False})
-        rsp = Response('http://www.scrapytest.org/503')
+        req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False})
+        rsp = Response("http://www.scrapytest.org/503")
 
         # first retry
         r = self.mw.process_response(req, rsp, self.spider)
         assert r is rsp
 
     def test_dont_retry_exc(self):
-        req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True})
+        req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
 
         r = self.mw.process_exception(req, DNSLookupError(), self.spider)
         assert r is None
 
     def test_503(self):
-        req = Request('http://www.scrapytest.org/503')
-        rsp = Response('http://www.scrapytest.org/503', body=b'', status=503)
+        req = Request("http://www.scrapytest.org/503")
+        rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
 
         # first retry
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
-        self.assertEqual(req.meta['retry_times'], 1)
+        self.assertEqual(req.meta["retry_times"], 1)
 
         # second retry
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
-        self.assertEqual(req.meta['retry_times'], 2)
+        self.assertEqual(req.meta["retry_times"], 2)
 
         # discard it
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
-        assert self.crawler.stats.get_value('retry/max_reached') == 1
-        assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2
-        assert self.crawler.stats.get_value('retry/count') == 2
+        assert self.crawler.stats.get_value("retry/max_reached") == 1
+        assert (
+            self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
+            == 2
+        )
+        assert self.crawler.stats.get_value("retry/count") == 2
 
     def test_twistederrors(self):
         exceptions = [
@@ -96,24 +99,27 @@ class RetryTest(unittest.TestCase):
         ]
 
         for exc in exceptions:
-            req = Request(f'http://www.scrapytest.org/{exc.__name__}')
-            self._test_retry_exception(req, exc('foo'))
+            req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+            self._test_retry_exception(req, exc("foo"))
 
         stats = self.crawler.stats
-        assert stats.get_value('retry/max_reached') == len(exceptions)
-        assert stats.get_value('retry/count') == len(exceptions) * 2
-        assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2
+        assert stats.get_value("retry/max_reached") == len(exceptions)
+        assert stats.get_value("retry/count") == len(exceptions) * 2
+        assert (
+            stats.get_value("retry/reason_count/twisted.internet.defer.TimeoutError")
+            == 2
+        )
 
     def _test_retry_exception(self, req, exception):
         # first retry
         req = self.mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
-        self.assertEqual(req.meta['retry_times'], 1)
+        self.assertEqual(req.meta["retry_times"], 1)
 
         # second retry
         req = self.mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
-        self.assertEqual(req.meta['retry_times'], 2)
+        self.assertEqual(req.meta["retry_times"], 2)
 
         # discard it
         req = self.mw.process_exception(req, exception, self.spider)
@@ -122,22 +128,22 @@ class RetryTest(unittest.TestCase):
 
 class MaxRetryTimesTest(unittest.TestCase):
 
-    invalid_url = 'http://www.scrapytest.org/invalid_url'
+    invalid_url = "http://www.scrapytest.org/invalid_url"
 
     def get_spider_and_middleware(self, settings=None):
         crawler = get_crawler(Spider, settings or {})
-        spider = crawler._create_spider('foo')
+        spider = crawler._create_spider("foo")
         middleware = RetryMiddleware.from_crawler(crawler)
         return spider, middleware
 
     def test_with_settings_zero(self):
         max_retry_times = 0
-        settings = {'RETRY_TIMES': max_retry_times}
+        settings = {"RETRY_TIMES": max_retry_times}
         spider, middleware = self.get_spider_and_middleware(settings)
         req = Request(self.invalid_url)
         self._test_retry(
             req,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             max_retry_times,
             spider=spider,
             middleware=middleware,
@@ -146,11 +152,11 @@ class MaxRetryTimesTest(unittest.TestCase):
     def test_with_metakey_zero(self):
         max_retry_times = 0
         spider, middleware = self.get_spider_and_middleware()
-        meta = {'max_retry_times': max_retry_times}
+        meta = {"max_retry_times": max_retry_times}
         req = Request(self.invalid_url, meta=meta)
         self._test_retry(
             req,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             max_retry_times,
             spider=spider,
             middleware=middleware,
@@ -158,12 +164,12 @@ class MaxRetryTimesTest(unittest.TestCase):
 
     def test_without_metakey(self):
         max_retry_times = 5
-        settings = {'RETRY_TIMES': max_retry_times}
+        settings = {"RETRY_TIMES": max_retry_times}
         spider, middleware = self.get_spider_and_middleware(settings)
         req = Request(self.invalid_url)
         self._test_retry(
             req,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             max_retry_times,
             spider=spider,
             middleware=middleware,
@@ -173,22 +179,22 @@ class MaxRetryTimesTest(unittest.TestCase):
         meta_max_retry_times = 3
         middleware_max_retry_times = 2
 
-        req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        req1 = Request(self.invalid_url, meta={"max_retry_times": meta_max_retry_times})
         req2 = Request(self.invalid_url)
 
-        settings = {'RETRY_TIMES': middleware_max_retry_times}
+        settings = {"RETRY_TIMES": middleware_max_retry_times}
         spider, middleware = self.get_spider_and_middleware(settings)
 
         self._test_retry(
             req1,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             meta_max_retry_times,
             spider=spider,
             middleware=middleware,
         )
         self._test_retry(
             req2,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             middleware_max_retry_times,
             spider=spider,
             middleware=middleware,
@@ -198,22 +204,22 @@ class MaxRetryTimesTest(unittest.TestCase):
         meta_max_retry_times = 4
         middleware_max_retry_times = 5
 
-        req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        req1 = Request(self.invalid_url, meta={"max_retry_times": meta_max_retry_times})
         req2 = Request(self.invalid_url)
 
-        settings = {'RETRY_TIMES': middleware_max_retry_times}
+        settings = {"RETRY_TIMES": middleware_max_retry_times}
         spider, middleware = self.get_spider_and_middleware(settings)
 
         self._test_retry(
             req1,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             meta_max_retry_times,
             spider=spider,
             middleware=middleware,
         )
         self._test_retry(
             req2,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             middleware_max_retry_times,
             spider=spider,
             middleware=middleware,
@@ -223,13 +229,13 @@ class MaxRetryTimesTest(unittest.TestCase):
         max_retry_times = 4
         spider, middleware = self.get_spider_and_middleware()
         meta = {
-            'max_retry_times': max_retry_times,
-            'dont_retry': True,
+            "max_retry_times": max_retry_times,
+            "dont_retry": True,
         }
         req = Request(self.invalid_url, meta=meta)
         self._test_retry(
             req,
-            DNSLookupError('foo'),
+            DNSLookupError("foo"),
             0,
             spider=spider,
             middleware=middleware,
@@ -256,13 +262,12 @@ class MaxRetryTimesTest(unittest.TestCase):
 
 
 class GetRetryRequestTest(unittest.TestCase):
-
     def get_spider(self, settings=None):
         crawler = get_crawler(Spider, settings or {})
-        return crawler._create_spider('foo')
+        return crawler._create_spider("foo")
 
     def test_basic_usage(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         with LogCapture() as log:
             new_request = get_retry_request(
@@ -273,10 +278,10 @@ class GetRetryRequestTest(unittest.TestCase):
         self.assertNotEqual(new_request, request)
         self.assertEqual(new_request.dont_filter, True)
         expected_retry_times = 1
-        self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
+        self.assertEqual(new_request.meta["retry_times"], expected_retry_times)
         self.assertEqual(new_request.priority, -1)
         expected_reason = "unspecified"
-        for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+        for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             self.assertEqual(spider.crawler.stats.get_value(stat), 1)
         log.check_present(
             (
@@ -288,7 +293,7 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_max_retries_reached(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         max_retry_times = 0
         with LogCapture() as log:
@@ -298,10 +303,7 @@ class GetRetryRequestTest(unittest.TestCase):
                 max_retry_times=max_retry_times,
             )
         self.assertEqual(new_request, None)
-        self.assertEqual(
-            spider.crawler.stats.get_value('retry/max_reached'),
-            1
-        )
+        self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1)
         failure_count = max_retry_times + 1
         expected_reason = "unspecified"
         log.check_present(
@@ -314,7 +316,7 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_one_retry(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         with LogCapture() as log:
             new_request = get_retry_request(
@@ -326,10 +328,10 @@ class GetRetryRequestTest(unittest.TestCase):
         self.assertNotEqual(new_request, request)
         self.assertEqual(new_request.dont_filter, True)
         expected_retry_times = 1
-        self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
+        self.assertEqual(new_request.meta["retry_times"], expected_retry_times)
         self.assertEqual(new_request.priority, -1)
         expected_reason = "unspecified"
-        for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+        for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             self.assertEqual(spider.crawler.stats.get_value(stat), 1)
         log.check_present(
             (
@@ -342,7 +344,7 @@ class GetRetryRequestTest(unittest.TestCase):
 
     def test_two_retries(self):
         spider = self.get_spider()
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         new_request = request
         max_retry_times = 2
         for index in range(max_retry_times):
@@ -356,10 +358,10 @@ class GetRetryRequestTest(unittest.TestCase):
             self.assertNotEqual(new_request, request)
             self.assertEqual(new_request.dont_filter, True)
             expected_retry_times = index + 1
-            self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
+            self.assertEqual(new_request.meta["retry_times"], expected_retry_times)
             self.assertEqual(new_request.priority, -expected_retry_times)
             expected_reason = "unspecified"
-            for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+            for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
                 value = spider.crawler.stats.get_value(stat)
                 self.assertEqual(value, expected_retry_times)
             log.check_present(
@@ -378,10 +380,7 @@ class GetRetryRequestTest(unittest.TestCase):
                 max_retry_times=max_retry_times,
             )
         self.assertEqual(new_request, None)
-        self.assertEqual(
-            spider.crawler.stats.get_value('retry/max_reached'),
-            1
-        )
+        self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1)
         failure_count = max_retry_times + 1
         expected_reason = "unspecified"
         log.check_present(
@@ -394,14 +393,14 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_no_spider(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         with self.assertRaises(TypeError):
             get_retry_request(request)  # pylint: disable=missing-kwoa
 
     def test_max_retry_times_setting(self):
         max_retry_times = 0
-        spider = self.get_spider({'RETRY_TIMES': max_retry_times})
-        request = Request('https://example.com')
+        spider = self.get_spider({"RETRY_TIMES": max_retry_times})
+        request = Request("https://example.com")
         new_request = get_retry_request(
             request,
             spider=spider,
@@ -410,9 +409,9 @@ class GetRetryRequestTest(unittest.TestCase):
 
     def test_max_retry_times_meta(self):
         max_retry_times = 0
-        spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
-        meta = {'max_retry_times': max_retry_times}
-        request = Request('https://example.com', meta=meta)
+        spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1})
+        meta = {"max_retry_times": max_retry_times}
+        request = Request("https://example.com", meta=meta)
         new_request = get_retry_request(
             request,
             spider=spider,
@@ -421,9 +420,9 @@ class GetRetryRequestTest(unittest.TestCase):
 
     def test_max_retry_times_argument(self):
         max_retry_times = 0
-        spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
-        meta = {'max_retry_times': max_retry_times + 1}
-        request = Request('https://example.com', meta=meta)
+        spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1})
+        meta = {"max_retry_times": max_retry_times + 1}
+        request = Request("https://example.com", meta=meta)
         new_request = get_retry_request(
             request,
             spider=spider,
@@ -433,8 +432,8 @@ class GetRetryRequestTest(unittest.TestCase):
 
     def test_priority_adjust_setting(self):
         priority_adjust = 1
-        spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust})
-        request = Request('https://example.com')
+        spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust})
+        request = Request("https://example.com")
         new_request = get_retry_request(
             request,
             spider=spider,
@@ -443,8 +442,8 @@ class GetRetryRequestTest(unittest.TestCase):
 
     def test_priority_adjust_argument(self):
         priority_adjust = 1
-        spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1})
-        request = Request('https://example.com')
+        spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1})
+        request = Request("https://example.com")
         new_request = get_retry_request(
             request,
             spider=spider,
@@ -453,9 +452,9 @@ class GetRetryRequestTest(unittest.TestCase):
         self.assertEqual(new_request.priority, priority_adjust)
 
     def test_log_extra_retry_success(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture(attributes=('spider',)) as log:
+        with LogCapture(attributes=("spider",)) as log:
             get_retry_request(
                 request,
                 spider=spider,
@@ -463,9 +462,9 @@ class GetRetryRequestTest(unittest.TestCase):
         log.check_present(spider)
 
     def test_log_extra_retries_exceeded(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture(attributes=('spider',)) as log:
+        with LogCapture(attributes=("spider",)) as log:
             get_retry_request(
                 request,
                 spider=spider,
@@ -474,9 +473,9 @@ class GetRetryRequestTest(unittest.TestCase):
         log.check_present(spider)
 
     def test_reason_string(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
-        expected_reason = 'because'
+        expected_reason = "because"
         with LogCapture() as log:
             get_retry_request(
                 request,
@@ -484,7 +483,7 @@ class GetRetryRequestTest(unittest.TestCase):
                 reason=expected_reason,
             )
         expected_retry_times = 1
-        for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+        for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             self.assertEqual(spider.crawler.stats.get_value(stat), 1)
         log.check_present(
             (
@@ -496,10 +495,10 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_reason_builtin_exception(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = NotImplementedError()
-        expected_reason_string = 'builtins.NotImplementedError'
+        expected_reason_string = "builtins.NotImplementedError"
         with LogCapture() as log:
             get_retry_request(
                 request,
@@ -508,7 +507,7 @@ class GetRetryRequestTest(unittest.TestCase):
             )
         expected_retry_times = 1
         stat = spider.crawler.stats.get_value(
-            f'retry/reason_count/{expected_reason_string}'
+            f"retry/reason_count/{expected_reason_string}"
         )
         self.assertEqual(stat, 1)
         log.check_present(
@@ -521,10 +520,10 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_reason_builtin_exception_class(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = NotImplementedError
-        expected_reason_string = 'builtins.NotImplementedError'
+        expected_reason_string = "builtins.NotImplementedError"
         with LogCapture() as log:
             get_retry_request(
                 request,
@@ -533,7 +532,7 @@ class GetRetryRequestTest(unittest.TestCase):
             )
         expected_retry_times = 1
         stat = spider.crawler.stats.get_value(
-            f'retry/reason_count/{expected_reason_string}'
+            f"retry/reason_count/{expected_reason_string}"
         )
         self.assertEqual(stat, 1)
         log.check_present(
@@ -546,10 +545,10 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_reason_custom_exception(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = IgnoreRequest()
-        expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
+        expected_reason_string = "scrapy.exceptions.IgnoreRequest"
         with LogCapture() as log:
             get_retry_request(
                 request,
@@ -558,7 +557,7 @@ class GetRetryRequestTest(unittest.TestCase):
             )
         expected_retry_times = 1
         stat = spider.crawler.stats.get_value(
-            f'retry/reason_count/{expected_reason_string}'
+            f"retry/reason_count/{expected_reason_string}"
         )
         self.assertEqual(stat, 1)
         log.check_present(
@@ -571,10 +570,10 @@ class GetRetryRequestTest(unittest.TestCase):
         )
 
     def test_reason_custom_exception_class(self):
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = IgnoreRequest
-        expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
+        expected_reason_string = "scrapy.exceptions.IgnoreRequest"
         with LogCapture() as log:
             get_retry_request(
                 request,
@@ -583,7 +582,7 @@ class GetRetryRequestTest(unittest.TestCase):
             )
         expected_retry_times = 1
         stat = spider.crawler.stats.get_value(
-            f'retry/reason_count/{expected_reason_string}'
+            f"retry/reason_count/{expected_reason_string}"
         )
         self.assertEqual(stat, 1)
         log.check_present(
@@ -626,7 +625,10 @@ class GetRetryRequestTest(unittest.TestCase):
             reason=expected_reason,
             stats_base_key=stats_key,
         )
-        for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"):
+        for stat in (
+            f"{stats_key}/count",
+            f"{stats_key}/reason_count/{expected_reason}",
+        ):
             self.assertEqual(spider.crawler.stats.get_value(stat), 1)
 
 
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 1460d88eb..ac08c6006 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -4,8 +4,10 @@ from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
 from twisted.trial import unittest
-from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
-                                                    logger as mw_module_logger)
+from scrapy.downloadermiddlewares.robotstxt import (
+    RobotsTxtMiddleware,
+    logger as mw_module_logger,
+)
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
@@ -13,7 +15,6 @@ from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
-
     def setUp(self):
         self.crawler = mock.MagicMock()
         self.crawler.settings = Settings()
@@ -24,12 +25,12 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
 
     def test_robotstxt_settings(self):
         self.crawler.settings = Settings()
-        self.crawler.settings.set('USER_AGENT', 'CustomAgent')
+        self.crawler.settings.set("USER_AGENT", "CustomAgent")
         self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler)
 
     def _get_successful_crawler(self):
         crawler = self.crawler
-        crawler.settings.set('ROBOTSTXT_OBEY', True)
+        crawler.settings.set("ROBOTSTXT_OBEY", True)
         ROBOTS = """
 User-Agent: *
 Disallow: /admin/
@@ -39,138 +40,179 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
 Disallow: /wiki/Käyttäjä:
 User-Agent: UnicödeBöt
 Disallow: /some/randome/page.html
-""".encode('utf-8')
-        response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
+""".encode(
+            "utf-8"
+        )
+        response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
 
         def return_response(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
             return deferred
+
         crawler.engine.download.side_effect = return_response
         return crawler
 
     def test_robotstxt(self):
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())
-        return DeferredList([
-            self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
-            self.assertIgnored(Request('http://site.local/admin/main'), middleware),
-            self.assertIgnored(Request('http://site.local/static/'), middleware),
-            self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
-            self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
-        ], fireOnOneErrback=True)
+        return DeferredList(
+            [
+                self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
+                self.assertIgnored(Request("http://site.local/admin/main"), middleware),
+                self.assertIgnored(Request("http://site.local/static/"), middleware),
+                self.assertIgnored(
+                    Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"),
+                    middleware,
+                ),
+                self.assertIgnored(
+                    Request("http://site.local/wiki/Käyttäjä:"), middleware
+                ),
+            ],
+            fireOnOneErrback=True,
+        )
 
     def test_robotstxt_ready_parser(self):
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())
-        d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
-        d.addCallback(lambda _: self.assertNotIgnored(Request('http://site.local/allowed'), middleware))
+        d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+        d.addCallback(
+            lambda _: self.assertNotIgnored(
+                Request("http://site.local/allowed"), middleware
+            )
+        )
         return d
 
     def test_robotstxt_meta(self):
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())
-        meta = {'dont_obey_robotstxt': True}
-        return DeferredList([
-            self.assertNotIgnored(Request('http://site.local/allowed', meta=meta), middleware),
-            self.assertNotIgnored(Request('http://site.local/admin/main', meta=meta), middleware),
-            self.assertNotIgnored(Request('http://site.local/static/', meta=meta), middleware)
-        ], fireOnOneErrback=True)
+        meta = {"dont_obey_robotstxt": True}
+        return DeferredList(
+            [
+                self.assertNotIgnored(
+                    Request("http://site.local/allowed", meta=meta), middleware
+                ),
+                self.assertNotIgnored(
+                    Request("http://site.local/admin/main", meta=meta), middleware
+                ),
+                self.assertNotIgnored(
+                    Request("http://site.local/static/", meta=meta), middleware
+                ),
+            ],
+            fireOnOneErrback=True,
+        )
 
     def _get_garbage_crawler(self):
         crawler = self.crawler
-        crawler.settings.set('ROBOTSTXT_OBEY', True)
-        response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')
+        crawler.settings.set("ROBOTSTXT_OBEY", True)
+        response = Response(
+            "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2"
+        )
 
         def return_response(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
             return deferred
+
         crawler.engine.download.side_effect = return_response
         return crawler
 
     def test_robotstxt_garbage(self):
         # garbage response should be discarded, equal 'allow all'
         middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
-        deferred = DeferredList([
-            self.assertNotIgnored(Request('http://site.local'), middleware),
-            self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
-            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware),
-            self.assertNotIgnored(Request('http://site.local/static/'), middleware)
-        ], fireOnOneErrback=True)
+        deferred = DeferredList(
+            [
+                self.assertNotIgnored(Request("http://site.local"), middleware),
+                self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
+                self.assertNotIgnored(
+                    Request("http://site.local/admin/main"), middleware
+                ),
+                self.assertNotIgnored(Request("http://site.local/static/"), middleware),
+            ],
+            fireOnOneErrback=True,
+        )
         return deferred
 
     def _get_emptybody_crawler(self):
         crawler = self.crawler
-        crawler.settings.set('ROBOTSTXT_OBEY', True)
-        response = Response('http://site.local/robots.txt')
+        crawler.settings.set("ROBOTSTXT_OBEY", True)
+        response = Response("http://site.local/robots.txt")
 
         def return_response(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
             return deferred
+
         crawler.engine.download.side_effect = return_response
         return crawler
 
     def test_robotstxt_empty_response(self):
         # empty response should equal 'allow all'
         middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
-        return DeferredList([
-            self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
-            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware),
-            self.assertNotIgnored(Request('http://site.local/static/'), middleware)
-        ], fireOnOneErrback=True)
+        return DeferredList(
+            [
+                self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
+                self.assertNotIgnored(
+                    Request("http://site.local/admin/main"), middleware
+                ),
+                self.assertNotIgnored(Request("http://site.local/static/"), middleware),
+            ],
+            fireOnOneErrback=True,
+        )
 
     def test_robotstxt_error(self):
-        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
-        err = error.DNSLookupError('Robotstxt address not found')
+        self.crawler.settings.set("ROBOTSTXT_OBEY", True)
+        err = error.DNSLookupError("Robotstxt address not found")
 
         def return_failure(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(err))
             return deferred
+
         self.crawler.engine.download.side_effect = return_failure
 
         middleware = RobotsTxtMiddleware(self.crawler)
         middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
-        deferred = middleware.process_request(Request('http://site.local'), None)
+        deferred = middleware.process_request(Request("http://site.local"), None)
         deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
         return deferred
 
     def test_robotstxt_immediate_error(self):
-        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
-        err = error.DNSLookupError('Robotstxt address not found')
+        self.crawler.settings.set("ROBOTSTXT_OBEY", True)
+        err = error.DNSLookupError("Robotstxt address not found")
 
         def immediate_failure(request):
             deferred = Deferred()
             deferred.errback(failure.Failure(err))
             return deferred
+
         self.crawler.engine.download.side_effect = immediate_failure
 
         middleware = RobotsTxtMiddleware(self.crawler)
-        return self.assertNotIgnored(Request('http://site.local'), middleware)
+        return self.assertNotIgnored(Request("http://site.local"), middleware)
 
     def test_ignore_robotstxt_request(self):
-        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+        self.crawler.settings.set("ROBOTSTXT_OBEY", True)
 
         def ignore_request(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))
             return deferred
+
         self.crawler.engine.download.side_effect = ignore_request
 
         middleware = RobotsTxtMiddleware(self.crawler)
         mw_module_logger.error = mock.MagicMock()
 
-        d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
+        d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
         d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
         return d
 
     def test_robotstxt_user_agent_setting(self):
         crawler = self._get_successful_crawler()
-        crawler.settings.set('ROBOTSTXT_USER_AGENT', 'Examplebot')
-        crawler.settings.set('USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64)')
+        crawler.settings.set("ROBOTSTXT_USER_AGENT", "Examplebot")
+        crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)")
         middleware = RobotsTxtMiddleware(crawler)
         rp = mock.MagicMock(return_value=True)
-        middleware.process_request_2(rp, Request('http://site.local/allowed'), None)
-        rp.allowed.assert_called_once_with('http://site.local/allowed', 'Examplebot')
+        middleware.process_request_2(rp, Request("http://site.local/allowed"), None)
+        rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
 
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
@@ -180,8 +222,9 @@ Disallow: /some/randome/page.html
 
     def assertIgnored(self, request, middleware):
         spider = None  # not actually used
-        return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),
-                                  IgnoreRequest)
+        return self.assertFailure(
+            maybeDeferred(middleware.process_request, request, spider), IgnoreRequest
+        )
 
 
 class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
@@ -190,7 +233,9 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
 
     def setUp(self):
         super().setUp()
-        self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
+        self.crawler.settings.set(
+            "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser"
+        )
 
 
 class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
@@ -199,4 +244,6 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
 
     def setUp(self):
         super().setUp()
-        self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')
+        self.crawler.settings.set(
+            "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser"
+        )
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 7d88ba4d2..39dfe9ab5 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -15,59 +15,57 @@ class MyException(Exception):
 
 
 class TestDownloaderStats(TestCase):
-
     def setUp(self):
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('scrapytest.org')
+        self.spider = self.crawler._create_spider("scrapytest.org")
         self.mw = DownloaderStats(self.crawler.stats)
 
         self.crawler.stats.open_spider(self.spider)
 
-        self.req = Request('http://scrapytest.org')
-        self.res = Response('scrapytest.org', status=400)
+        self.req = Request("http://scrapytest.org")
+        self.res = Response("scrapytest.org", status=400)
 
     def assertStatsEqual(self, key, value):
         self.assertEqual(
             self.crawler.stats.get_value(key, spider=self.spider),
             value,
-            str(self.crawler.stats.get_stats(self.spider))
+            str(self.crawler.stats.get_stats(self.spider)),
         )
 
     def test_process_request(self):
         self.mw.process_request(self.req, self.spider)
-        self.assertStatsEqual('downloader/request_count', 1)
+        self.assertStatsEqual("downloader/request_count", 1)
 
     def test_process_response(self):
         self.mw.process_response(self.req, self.res, self.spider)
-        self.assertStatsEqual('downloader/response_count', 1)
+        self.assertStatsEqual("downloader/response_count", 1)
 
     def test_response_len(self):
-        body = (b'', b'not_empty')  # empty/notempty body
-        headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'})  # 0 headers, 1h and 2h
+        body = (b"", b"not_empty")  # empty/notempty body
+        headers = (
+            {},
+            {"lang": "en"},
+            {"lang": "en", "User-Agent": "scrapy"},
+        )  # 0 headers, 1h and 2h
         test_responses = [  # form test responses with all combinations of body/headers
-            Response(
-                url='scrapytest.org',
-                status=200,
-                body=r[0],
-                headers=r[1]
-            )
+            Response(url="scrapytest.org", status=200, body=r[0], headers=r[1])
             for r in product(body, headers)
         ]
         for test_response in test_responses:
-            self.crawler.stats.set_value('downloader/response_bytes', 0)
+            self.crawler.stats.set_value("downloader/response_bytes", 0)
             self.mw.process_response(self.req, test_response, self.spider)
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", ScrapyDeprecationWarning)
                 resp_size = len(response_httprepr(test_response))
-            self.assertStatsEqual('downloader/response_bytes', resp_size)
+            self.assertStatsEqual("downloader/response_bytes", resp_size)
 
     def test_process_exception(self):
         self.mw.process_exception(self.req, MyException(), self.spider)
-        self.assertStatsEqual('downloader/exception_count', 1)
+        self.assertStatsEqual("downloader/exception_count", 1)
         self.assertStatsEqual(
-            'downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException',
-            1
+            "downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException",
+            1,
         )
 
     def tearDown(self):
-        self.crawler.stats.close_spider(self.spider, '')
+        self.crawler.stats.close_spider(self.spider, "")
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index a286764fd..927f0ae4d 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -7,48 +7,48 @@ from scrapy.utils.test import get_crawler
 
 
 class UserAgentMiddlewareTest(TestCase):
-
     def get_spider_and_mw(self, default_useragent):
-        crawler = get_crawler(Spider, {'USER_AGENT': default_useragent})
-        spider = crawler._create_spider('foo')
+        crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
+        spider = crawler._create_spider("foo")
         return spider, UserAgentMiddleware.from_crawler(crawler)
 
     def test_default_agent(self):
-        spider, mw = self.get_spider_and_mw('default_useragent')
-        req = Request('http://scrapytest.org/')
+        spider, mw = self.get_spider_and_mw("default_useragent")
+        req = Request("http://scrapytest.org/")
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.headers['User-Agent'], b'default_useragent')
+        self.assertEqual(req.headers["User-Agent"], b"default_useragent")
 
     def test_remove_agent(self):
         # settings UESR_AGENT to None should remove the user agent
-        spider, mw = self.get_spider_and_mw('default_useragent')
+        spider, mw = self.get_spider_and_mw("default_useragent")
         spider.user_agent = None
         mw.spider_opened(spider)
-        req = Request('http://scrapytest.org/')
+        req = Request("http://scrapytest.org/")
         assert mw.process_request(req, spider) is None
-        assert req.headers.get('User-Agent') is None
+        assert req.headers.get("User-Agent") is None
 
     def test_spider_agent(self):
-        spider, mw = self.get_spider_and_mw('default_useragent')
-        spider.user_agent = 'spider_useragent'
+        spider, mw = self.get_spider_and_mw("default_useragent")
+        spider.user_agent = "spider_useragent"
         mw.spider_opened(spider)
-        req = Request('http://scrapytest.org/')
+        req = Request("http://scrapytest.org/")
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.headers['User-Agent'], b'spider_useragent')
+        self.assertEqual(req.headers["User-Agent"], b"spider_useragent")
 
     def test_header_agent(self):
-        spider, mw = self.get_spider_and_mw('default_useragent')
-        spider.user_agent = 'spider_useragent'
+        spider, mw = self.get_spider_and_mw("default_useragent")
+        spider.user_agent = "spider_useragent"
         mw.spider_opened(spider)
-        req = Request('http://scrapytest.org/',
-                      headers={'User-Agent': 'header_useragent'})
+        req = Request(
+            "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
+        )
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.headers['User-Agent'], b'header_useragent')
+        self.assertEqual(req.headers["User-Agent"], b"header_useragent")
 
     def test_no_agent(self):
         spider, mw = self.get_spider_and_mw(None)
         spider.user_agent = None
         mw.spider_opened(spider)
-        req = Request('http://scrapytest.org/')
+        req = Request("http://scrapytest.org/")
         assert mw.process_request(req, spider) is None
-        assert 'User-Agent' not in req.headers
+        assert "User-Agent" not in req.headers
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index e028dea4d..4019012d1 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -25,59 +25,62 @@ def _get_dupefilter(*, crawler=None, settings=None, open=True):
 
 
 class FromCrawlerRFPDupeFilter(RFPDupeFilter):
-
     @classmethod
     def from_crawler(cls, crawler):
         df = super().from_crawler(crawler)
-        df.method = 'from_crawler'
+        df.method = "from_crawler"
         return df
 
 
 class FromSettingsRFPDupeFilter(RFPDupeFilter):
-
     @classmethod
     def from_settings(cls, settings, *, fingerprinter=None):
         df = super().from_settings(settings, fingerprinter=fingerprinter)
-        df.method = 'from_settings'
+        df.method = "from_settings"
         return df
 
 
 class DirectDupeFilter:
-    method = 'n/a'
+    method = "n/a"
 
 
 class RFPDupeFilterTest(unittest.TestCase):
-
     def test_df_from_crawler_scheduler(self):
-        settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
-                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+        settings = {
+            "DUPEFILTER_DEBUG": True,
+            "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+        }
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
-        self.assertEqual(scheduler.df.method, 'from_crawler')
+        self.assertEqual(scheduler.df.method, "from_crawler")
 
     def test_df_from_settings_scheduler(self):
-        settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
-                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+        settings = {
+            "DUPEFILTER_DEBUG": True,
+            "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter,
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+        }
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
-        self.assertEqual(scheduler.df.method, 'from_settings')
+        self.assertEqual(scheduler.df.method, "from_settings")
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
-                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+        settings = {
+            "DUPEFILTER_CLASS": DirectDupeFilter,
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+        }
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
-        self.assertEqual(scheduler.df.method, 'n/a')
+        self.assertEqual(scheduler.df.method, "n/a")
 
     def test_filter(self):
         dupefilter = _get_dupefilter()
-        r1 = Request('http://scrapytest.org/1')
-        r2 = Request('http://scrapytest.org/2')
-        r3 = Request('http://scrapytest.org/2')
+        r1 = Request("http://scrapytest.org/1")
+        r2 = Request("http://scrapytest.org/2")
+        r3 = Request("http://scrapytest.org/2")
 
         assert not dupefilter.request_seen(r1)
         assert dupefilter.request_seen(r1)
@@ -85,23 +88,23 @@ class RFPDupeFilterTest(unittest.TestCase):
         assert not dupefilter.request_seen(r2)
         assert dupefilter.request_seen(r3)
 
-        dupefilter.close('finished')
+        dupefilter.close("finished")
 
     def test_dupefilter_path(self):
-        r1 = Request('http://scrapytest.org/1')
-        r2 = Request('http://scrapytest.org/2')
+        r1 = Request("http://scrapytest.org/1")
+        r2 = Request("http://scrapytest.org/2")
 
         path = tempfile.mkdtemp()
         try:
-            df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+            df = _get_dupefilter(settings={"JOBDIR": path}, open=False)
             try:
                 df.open()
                 assert not df.request_seen(r1)
                 assert df.request_seen(r1)
             finally:
-                df.close('finished')
+                df.close("finished")
 
-            df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+            df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False)
             assert df != df2
             try:
                 df2.open()
@@ -109,7 +112,7 @@ class RFPDupeFilterTest(unittest.TestCase):
                 assert not df2.request_seen(r2)
                 assert df2.request_seen(r2)
             finally:
-                df2.close('finished')
+                df2.close("finished")
         finally:
             shutil.rmtree(path)
 
@@ -119,147 +122,156 @@ class RFPDupeFilterTest(unittest.TestCase):
 
         """
         dupefilter = _get_dupefilter()
-        r1 = Request('http://scrapytest.org/index.html')
-        r2 = Request('http://scrapytest.org/INDEX.html')
+        r1 = Request("http://scrapytest.org/index.html")
+        r2 = Request("http://scrapytest.org/INDEX.html")
 
         assert not dupefilter.request_seen(r1)
         assert not dupefilter.request_seen(r2)
 
-        dupefilter.close('finished')
+        dupefilter.close("finished")
 
         class RequestFingerprinter:
-
             def fingerprint(self, request):
                 fp = hashlib.sha1()
                 fp.update(to_bytes(request.url.lower()))
                 return fp.digest()
 
-        settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
+        settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter}
         case_insensitive_dupefilter = _get_dupefilter(settings=settings)
 
         assert not case_insensitive_dupefilter.request_seen(r1)
         assert case_insensitive_dupefilter.request_seen(r2)
 
-        case_insensitive_dupefilter.close('finished')
+        case_insensitive_dupefilter.close("finished")
 
     def test_seenreq_newlines(self):
-        """ Checks against adding duplicate \r to
-        line endings on Windows platforms. """
+        """Checks against adding duplicate \r to
+        line endings on Windows platforms."""
 
-        r1 = Request('http://scrapytest.org/1')
+        r1 = Request("http://scrapytest.org/1")
 
         path = tempfile.mkdtemp()
-        crawler = get_crawler(settings_dict={'JOBDIR': path})
+        crawler = get_crawler(settings_dict={"JOBDIR": path})
         try:
             scheduler = Scheduler.from_crawler(crawler)
             df = scheduler.df
             df.open()
             df.request_seen(r1)
-            df.close('finished')
+            df.close("finished")
 
-            with Path(path, 'requests.seen').open('rb') as seen_file:
+            with Path(path, "requests.seen").open("rb") as seen_file:
                 line = next(seen_file).decode()
-                assert not line.endswith('\r\r\n')
-                if sys.platform == 'win32':
-                    assert line.endswith('\r\n')
+                assert not line.endswith("\r\r\n")
+                if sys.platform == "win32":
+                    assert line.endswith("\r\n")
                 else:
-                    assert line.endswith('\n')
+                    assert line.endswith("\n")
 
         finally:
             shutil.rmtree(path)
 
     def test_log(self):
         with LogCapture() as log:
-            settings = {'DUPEFILTER_DEBUG': False,
-                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
-                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+            settings = {
+                "DUPEFILTER_DEBUG": False,
+                "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+            }
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request('http://scrapytest.org/index.html')
-            r2 = Request('http://scrapytest.org/index.html')
+            r1 = Request("http://scrapytest.org/index.html")
+            r2 = Request("http://scrapytest.org/index.html")
 
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            assert crawler.stats.get_value("dupefilter/filtered") == 2
             log.check_present(
                 (
-                    'scrapy.dupefilters',
-                    'DEBUG',
-                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more'
-                    ' duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
+                    "scrapy.dupefilters",
+                    "DEBUG",
+                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
+                    " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",
                 )
             )
 
-            dupefilter.close('finished')
+            dupefilter.close("finished")
 
     def test_log_debug(self):
         with LogCapture() as log:
-            settings = {'DUPEFILTER_DEBUG': True,
-                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
-                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+            settings = {
+                "DUPEFILTER_DEBUG": True,
+                "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+            }
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request('http://scrapytest.org/index.html')
-            r2 = Request('http://scrapytest.org/index.html',
-                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
+            r1 = Request("http://scrapytest.org/index.html")
+            r2 = Request(
+                "http://scrapytest.org/index.html",
+                headers={"Referer": "http://scrapytest.org/INDEX.html"},
+            )
 
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            assert crawler.stats.get_value("dupefilter/filtered") == 2
             log.check_present(
                 (
-                    'scrapy.dupefilters',
-                    'DEBUG',
-                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+                    "scrapy.dupefilters",
+                    "DEBUG",
+                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
                 )
             )
             log.check_present(
                 (
-                    'scrapy.dupefilters',
-                    'DEBUG',
-                    'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                    ' (referer: http://scrapytest.org/INDEX.html)'
+                    "scrapy.dupefilters",
+                    "DEBUG",
+                    "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
+                    " (referer: http://scrapytest.org/INDEX.html)",
                 )
             )
 
-            dupefilter.close('finished')
+            dupefilter.close("finished")
 
     def test_log_debug_default_dupefilter(self):
         with LogCapture() as log:
-            settings = {'DUPEFILTER_DEBUG': True,
-                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+            settings = {
+                "DUPEFILTER_DEBUG": True,
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+            }
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request('http://scrapytest.org/index.html')
-            r2 = Request('http://scrapytest.org/index.html',
-                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
+            r1 = Request("http://scrapytest.org/index.html")
+            r2 = Request(
+                "http://scrapytest.org/index.html",
+                headers={"Referer": "http://scrapytest.org/INDEX.html"},
+            )
 
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            assert crawler.stats.get_value("dupefilter/filtered") == 2
             log.check_present(
                 (
-                    'scrapy.dupefilters',
-                    'DEBUG',
-                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+                    "scrapy.dupefilters",
+                    "DEBUG",
+                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
                 )
             )
             log.check_present(
                 (
-                    'scrapy.dupefilters',
-                    'DEBUG',
-                    'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                    ' (referer: http://scrapytest.org/INDEX.html)'
+                    "scrapy.dupefilters",
+                    "DEBUG",
+                    "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
+                    " (referer: http://scrapytest.org/INDEX.html)",
                 )
             )
 
-            dupefilter.close('finished')
+            dupefilter.close("finished")
diff --git a/tests/test_engine.py b/tests/test_engine.py
index aa3313659..7b861332e 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -81,11 +81,11 @@ class TestSpider(Spider):
         adapter = ItemAdapter(self.item_cls())
         m = self.name_re.search(response.text)
         if m:
-            adapter['name'] = m.group(1)
-        adapter['url'] = response.url
+            adapter["name"] = m.group(1)
+        adapter["url"] = response.url
         m = self.price_re.search(response.text)
         if m:
-            adapter['price'] = m.group(1)
+            adapter["price"] = m.group(1)
         return adapter.item
 
 
@@ -136,8 +136,10 @@ def start_test_site(debug=False):
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
-        print(f"Test server running at http://localhost:{port.getHost().port}/ "
-              "- hit Ctrl-C to finish.")
+        print(
+            f"Test server running at http://localhost:{port.getHost().port}/ "
+            "- hit Ctrl-C to finish."
+        )
     return port
 
 
@@ -169,7 +171,7 @@ class CrawlerRun:
         ]
 
         for name, signal in vars(signals).items():
-            if not name.startswith('_'):
+            if not name.startswith("_"):
                 dispatcher.connect(self.record_signal, signal)
 
         self.crawler = get_crawler(self.spider_class)
@@ -179,8 +181,12 @@ class CrawlerRun:
         self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
-        self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
-        self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
+        self.crawler.signals.connect(
+            self.request_reached, signals.request_reached_downloader
+        )
+        self.crawler.signals.connect(
+            self.response_downloaded, signals.response_downloaded
+        )
         self.crawler.crawl(start_urls=start_urls)
         self.spider = self.crawler.spider
 
@@ -191,7 +197,7 @@ class CrawlerRun:
     def stop(self):
         self.port.stopListening()  # FIXME: wait for this Deferred
         for name, signal in vars(signals).items():
-            if not name.startswith('_'):
+            if not name.startswith("_"):
                 disconnect_all(signal)
         self.deferred.callback(None)
         return self.crawler.stop()
@@ -230,8 +236,8 @@ class CrawlerRun:
     def record_signal(self, *args, **kwargs):
         """Record a signal and its parameters"""
         signalargs = kwargs.copy()
-        sig = signalargs.pop('signal')
-        signalargs.pop('sender', None)
+        sig = signalargs.pop("signal")
+        signalargs.pop("sender", None)
         self.signals_caught[sig] = signalargs
 
 
@@ -239,7 +245,12 @@ class EngineTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_crawler(self):
 
-        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+        for spider in (
+            TestSpider,
+            DictItemsSpider,
+            AttrsItemsSpider,
+            DataClassItemsSpider,
+        ):
             run = CrawlerRun(spider)
             yield run.run()
             self._assert_visited_urls(run)
@@ -266,20 +277,30 @@ class EngineTest(unittest.TestCase):
     def test_crawler_change_close_reason_on_idle(self):
         run = CrawlerRun(ChangeCloseReasonSpider)
         yield run.run()
-        self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},
-                         run.signals_caught[signals.spider_closed])
+        self.assertEqual(
+            {"spider": run.spider, "reason": "custom_reason"},
+            run.signals_caught[signals.spider_closed],
+        )
 
     def _assert_visited_urls(self, run: CrawlerRun):
-        must_be_visited = ["/", "/redirect", "/redirected",
-                           "/item1.html", "/item2.html", "/item999.html"]
+        must_be_visited = [
+            "/",
+            "/redirect",
+            "/redirected",
+            "/item1.html",
+            "/item2.html",
+            "/item999.html",
+        ]
         urls_visited = {rp[0].url for rp in run.respplug}
         urls_expected = {run.geturl(p) for p in must_be_visited}
-        assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
+        assert (
+            urls_expected <= urls_visited
+        ), f"URLs not visited: {list(urls_expected - urls_visited)}"
 
     def _assert_scheduled_requests(self, run: CrawlerRun, count=None):
         self.assertEqual(count, len(run.reqplug))
 
-        paths_expected = ['/item999.html', '/item2.html', '/item1.html']
+        paths_expected = ["/item999.html", "/item2.html", "/item1.html"]
 
         urls_requested = {rq[0].url for rq in run.reqplug}
         urls_expected = {run.geturl(p) for p in paths_expected}
@@ -287,10 +308,10 @@ class EngineTest(unittest.TestCase):
         scheduled_requests_count = len(run.reqplug)
         dropped_requests_count = len(run.reqdropped)
         responses_count = len(run.respplug)
-        self.assertEqual(scheduled_requests_count,
-                         dropped_requests_count + responses_count)
-        self.assertEqual(len(run.reqreached),
-                         responses_count)
+        self.assertEqual(
+            scheduled_requests_count, dropped_requests_count + responses_count
+        )
+        self.assertEqual(len(run.reqreached), responses_count)
 
     def _assert_dropped_requests(self, run: CrawlerRun):
         self.assertEqual(len(run.reqdropped), 1)
@@ -301,9 +322,9 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(count, len(run.reqreached))
 
         for response, _ in run.respplug:
-            if run.getpath(response.url) == '/item999.html':
+            if run.getpath(response.url) == "/item999.html":
                 self.assertEqual(404, response.status)
-            if run.getpath(response.url) == '/redirect':
+            if run.getpath(response.url) == "/redirect":
                 self.assertEqual(302, response.status)
 
     def _assert_items_error(self, run: CrawlerRun):
@@ -312,25 +333,25 @@ class EngineTest(unittest.TestCase):
             self.assertEqual(failure.value.__class__, ZeroDivisionError)
             self.assertEqual(spider, run.spider)
 
-            self.assertEqual(item['url'], response.url)
-            if 'item1.html' in item['url']:
-                self.assertEqual('Item 1 name', item['name'])
-                self.assertEqual('100', item['price'])
-            if 'item2.html' in item['url']:
-                self.assertEqual('Item 2 name', item['name'])
-                self.assertEqual('200', item['price'])
+            self.assertEqual(item["url"], response.url)
+            if "item1.html" in item["url"]:
+                self.assertEqual("Item 1 name", item["name"])
+                self.assertEqual("100", item["price"])
+            if "item2.html" in item["url"]:
+                self.assertEqual("Item 2 name", item["name"])
+                self.assertEqual("200", item["price"])
 
     def _assert_scraped_items(self, run: CrawlerRun):
         self.assertEqual(2, len(run.itemresp))
         for item, response in run.itemresp:
             item = ItemAdapter(item)
-            self.assertEqual(item['url'], response.url)
-            if 'item1.html' in item['url']:
-                self.assertEqual('Item 1 name', item['name'])
-                self.assertEqual('100', item['price'])
-            if 'item2.html' in item['url']:
-                self.assertEqual('Item 2 name', item['name'])
-                self.assertEqual('200', item['price'])
+            self.assertEqual(item["url"], response.url)
+            if "item1.html" in item["url"]:
+                self.assertEqual("Item 1 name", item["name"])
+                self.assertEqual("100", item["price"])
+            if "item2.html" in item["url"]:
+                self.assertEqual("Item 2 name", item["name"])
+                self.assertEqual("200", item["price"])
 
     def _assert_headers_received(self, run: CrawlerRun):
         for headers in run.headers.values():
@@ -351,17 +372,17 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
             elif run.getpath(request.url) == "/redirected":
                 self.assertEqual(joined_data, b"Redirected here")
-            elif run.getpath(request.url) == '/redirect':
+            elif run.getpath(request.url) == "/redirect":
                 self.assertEqual(
                     joined_data,
                     b"\n<html>\n"
                     b"    <head>\n"
-                    b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
+                    b'        <meta http-equiv="refresh" content="0;URL=/redirected">\n'
                     b"    </head>\n"
-                    b"    <body bgcolor=\"#FFFFFF\" text=\"#000000\">\n"
-                    b"    <a href=\"/redirected\">click here</a>\n"
+                    b'    <body bgcolor="#FFFFFF" text="#000000">\n'
+                    b'    <a href="/redirected">click here</a>\n'
                     b"    </body>\n"
-                    b"</html>\n"
+                    b"</html>\n",
                 )
             elif run.getpath(request.url) == "/tem999.html":
                 self.assertEqual(
@@ -372,7 +393,7 @@ class EngineTest(unittest.TestCase):
                     b"    <h1>No Such Resource</h1>\n"
                     b"    <p>File not found.</p>\n"
                     b"  </body>\n"
-                    b"</html>\n"
+                    b"</html>\n",
                 )
             elif run.getpath(request.url) == "/numbers":
                 # signal was fired multiple times
@@ -389,12 +410,16 @@ class EngineTest(unittest.TestCase):
         assert signals.spider_closed in run.signals_caught
         assert signals.headers_received in run.signals_caught
 
-        self.assertEqual({'spider': run.spider},
-                         run.signals_caught[signals.spider_opened])
-        self.assertEqual({'spider': run.spider},
-                         run.signals_caught[signals.spider_idle])
-        self.assertEqual({'spider': run.spider, 'reason': 'finished'},
-                         run.signals_caught[signals.spider_closed])
+        self.assertEqual(
+            {"spider": run.spider}, run.signals_caught[signals.spider_opened]
+        )
+        self.assertEqual(
+            {"spider": run.spider}, run.signals_caught[signals.spider_idle]
+        )
+        self.assertEqual(
+            {"spider": run.spider, "reason": "finished"},
+            run.signals_caught[signals.spider_closed],
+        )
 
     @defer.inlineCallbacks
     def test_close_downloader(self):
@@ -415,9 +440,11 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_close_spiders_downloader(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="ExecutionEngine.open_spiders is deprecated, "
-                                "please use ExecutionEngine.spider instead"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="ExecutionEngine.open_spiders is deprecated, "
+            "please use ExecutionEngine.spider instead",
+        ):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             yield e.open_spider(TestSpider(), [])
             self.assertEqual(len(e.open_spiders), 1)
@@ -426,9 +453,11 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_close_engine_spiders_downloader(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="ExecutionEngine.open_spiders is deprecated, "
-                                "please use ExecutionEngine.spider instead"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="ExecutionEngine.open_spiders is deprecated, "
+            "please use ExecutionEngine.spider instead",
+        ):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             yield e.open_spider(TestSpider(), [])
             e.start()
@@ -439,9 +468,11 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawl_deprecated_spider_arg(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="Passing a 'spider' argument to "
-                                "ExecutionEngine.crawl is deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Passing a 'spider' argument to "
+            "ExecutionEngine.crawl is deprecated",
+        ):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
@@ -451,9 +482,11 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download_deprecated_spider_arg(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="Passing a 'spider' argument to "
-                                "ExecutionEngine.download is deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Passing a 'spider' argument to "
+            "ExecutionEngine.download is deprecated",
+        ):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
@@ -463,9 +496,11 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_deprecated_schedule(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="ExecutionEngine.schedule is deprecated, please use "
-                                "ExecutionEngine.crawl or ExecutionEngine.download instead"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="ExecutionEngine.schedule is deprecated, please use "
+            "ExecutionEngine.crawl or ExecutionEngine.download instead",
+        ):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
@@ -475,8 +510,9 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_deprecated_has_capacity(self):
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="ExecutionEngine.has_capacity is deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated"
+        ):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             self.assertTrue(e.has_capacity())
             spider = TestSpider()
@@ -489,14 +525,14 @@ class EngineTest(unittest.TestCase):
     def test_short_timeout(self):
         args = (
             sys.executable,
-            '-m',
-            'scrapy.cmdline',
-            'fetch',
-            '-s',
-            'CLOSESPIDER_TIMEOUT=0.001',
-            '-s',
-            'LOG_LEVEL=DEBUG',
-            'http://toscrape.com',
+            "-m",
+            "scrapy.cmdline",
+            "fetch",
+            "-s",
+            "CLOSESPIDER_TIMEOUT=0.001",
+            "-s",
+            "LOG_LEVEL=DEBUG",
+            "http://toscrape.com",
         )
         p = subprocess.Popen(
             args,
@@ -506,7 +542,7 @@ class EngineTest(unittest.TestCase):
         def kill_proc():
             p.kill()
             p.communicate()
-            assert False, 'Command took too much time to complete'
+            assert False, "Command took too much time to complete"
 
         timer = Timer(15, kill_proc)
         try:
@@ -515,10 +551,10 @@ class EngineTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        self.assertNotIn(b'Traceback', stderr)
+        self.assertNotIn(b"Traceback", stderr)
 
 
 if __name__ == "__main__":
-    if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
+    if len(sys.argv) > 1 and sys.argv[1] == "runserver":
         start_test_site(debug=True)
         reactor.run()
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 933e4067d..fb8dd4313 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -22,22 +22,39 @@ class BytesReceivedCrawlerRun(CrawlerRun):
 class BytesReceivedEngineTest(EngineTest):
     @defer.inlineCallbacks
     def test_crawler(self):
-        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+        for spider in (
+            TestSpider,
+            DictItemsSpider,
+            AttrsItemsSpider,
+            DataClassItemsSpider,
+        ):
             run = BytesReceivedCrawlerRun(spider)
             with LogCapture() as log:
                 yield run.run()
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{run.portno}/redirected> "
-                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{run.portno}/> "
-                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{run.portno}/numbers> "
-                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+                log.check_present(
+                    (
+                        "scrapy.core.downloader.handlers.http11",
+                        "DEBUG",
+                        f"Download stopped for <GET http://localhost:{run.portno}/redirected> "
+                        "from signal handler BytesReceivedCrawlerRun.bytes_received",
+                    )
+                )
+                log.check_present(
+                    (
+                        "scrapy.core.downloader.handlers.http11",
+                        "DEBUG",
+                        f"Download stopped for <GET http://localhost:{run.portno}/> "
+                        "from signal handler BytesReceivedCrawlerRun.bytes_received",
+                    )
+                )
+                log.check_present(
+                    (
+                        "scrapy.core.downloader.handlers.http11",
+                        "DEBUG",
+                        f"Download stopped for <GET http://localhost:{run.portno}/numbers> "
+                        "from signal handler BytesReceivedCrawlerRun.bytes_received",
+                    )
+                )
             self._assert_visited_urls(run)
             self._assert_scheduled_requests(run, count=9)
             self._assert_downloaded_responses(run, count=9)
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index 8975d0e3f..93437559d 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -22,22 +22,39 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
 class HeadersReceivedEngineTest(EngineTest):
     @defer.inlineCallbacks
     def test_crawler(self):
-        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+        for spider in (
+            TestSpider,
+            DictItemsSpider,
+            AttrsItemsSpider,
+            DataClassItemsSpider,
+        ):
             run = HeadersReceivedCrawlerRun(spider)
             with LogCapture() as log:
                 yield run.run()
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"
-                                   " signal handler HeadersReceivedCrawlerRun.headers_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{run.portno}/> from signal"
-                                   " handler HeadersReceivedCrawlerRun.headers_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"
-                                   " signal handler HeadersReceivedCrawlerRun.headers_received"))
+                log.check_present(
+                    (
+                        "scrapy.core.downloader.handlers.http11",
+                        "DEBUG",
+                        f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"
+                        " signal handler HeadersReceivedCrawlerRun.headers_received",
+                    )
+                )
+                log.check_present(
+                    (
+                        "scrapy.core.downloader.handlers.http11",
+                        "DEBUG",
+                        f"Download stopped for <GET http://localhost:{run.portno}/> from signal"
+                        " handler HeadersReceivedCrawlerRun.headers_received",
+                    )
+                )
+                log.check_present(
+                    (
+                        "scrapy.core.downloader.handlers.http11",
+                        "DEBUG",
+                        f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"
+                        " signal handler HeadersReceivedCrawlerRun.headers_received",
+                    )
+                )
             self._assert_visited_urls(run)
             self._assert_downloaded_responses(run, count=6)
             self._assert_signals_caught(run)
@@ -51,4 +68,6 @@ class HeadersReceivedEngineTest(EngineTest):
         must_be_visited = ["/", "/redirect", "/redirected"]
         urls_visited = {rp[0].url for rp in run.respplug}
         urls_expected = {run.geturl(p) for p in must_be_visited}
-        assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
+        assert (
+            urls_expected <= urls_visited
+        ), f"URLs not visited: {list(urls_expected - urls_visited)}"
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 86c5d8b5e..7689045b7 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -16,9 +16,15 @@ from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.exporters import (
-    BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
-    XmlItemExporter, JsonLinesItemExporter, JsonItemExporter,
-    PythonItemExporter, MarshalItemExporter
+    BaseItemExporter,
+    PprintItemExporter,
+    PickleItemExporter,
+    CsvItemExporter,
+    XmlItemExporter,
+    JsonLinesItemExporter,
+    JsonItemExporter,
+    PythonItemExporter,
+    MarshalItemExporter,
 )
 
 
@@ -54,7 +60,7 @@ class BaseItemExporterTest(unittest.TestCase):
     custom_field_item_class = CustomFieldItem
 
     def setUp(self):
-        self.i = self.item_class(name='John\xa3', age='22')
+        self.i = self.item_class(name="John\xa3", age="22")
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -71,10 +77,10 @@ class BaseItemExporterTest(unittest.TestCase):
 
     def _get_nonstring_types_item(self):
         return {
-            'boolean': False,
-            'number': 22,
-            'time': datetime(2015, 1, 1, 1, 1, 1),
-            'float': 3.14,
+            "boolean": False,
+            "number": 22,
+            "time": datetime(2015, 1, 1, 1, 1, 1),
+            "float": 3.14,
         }
 
     def assertItemExportWorks(self, item):
@@ -95,35 +101,36 @@ class BaseItemExporterTest(unittest.TestCase):
 
     def test_serialize_field(self):
         a = ItemAdapter(self.i)
-        res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
-        self.assertEqual(res, 'John\xa3')
+        res = self.ie.serialize_field(a.get_field_meta("name"), "name", a["name"])
+        self.assertEqual(res, "John\xa3")
 
-        res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
-        self.assertEqual(res, '22')
+        res = self.ie.serialize_field(a.get_field_meta("age"), "age", a["age"])
+        self.assertEqual(res, "22")
 
     def test_fields_to_export(self):
-        ie = self._get_exporter(fields_to_export=['name'])
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
+        ie = self._get_exporter(fields_to_export=["name"])
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)), [("name", "John\xa3")]
+        )
 
-        ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
+        ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1")
         _, name = list(ie._get_serialized_fields(self.i))[0]
         assert isinstance(name, str)
-        self.assertEqual(name, 'John\xa3')
+        self.assertEqual(name, "John\xa3")
 
-        ie = self._get_exporter(
-            fields_to_export={'name': '名稱'}
-        )
-        self.assertEqual(
-            list(ie._get_serialized_fields(self.i)),
-            [('名稱', 'John\xa3')]
-        )
+        ie = self._get_exporter(fields_to_export={"name": "名稱"})
+        self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")])
 
     def test_field_custom_serializer(self):
-        i = self.custom_field_item_class(name='John\xa3', age='22')
+        i = self.custom_field_item_class(name="John\xa3", age="22")
         a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
-        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
+        self.assertEqual(
+            ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John\xa3"
+        )
+        self.assertEqual(
+            ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "24"
+        )
 
 
 class BaseItemExporterDataclassTest(BaseItemExporterTest):
@@ -137,54 +144,63 @@ class PythonItemExporterTest(BaseItemExporterTest):
 
     def test_invalid_option(self):
         with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
-            PythonItemExporter(invalid_option='something')
+            PythonItemExporter(invalid_option="something")
 
     def test_nested_item(self):
-        i1 = self.item_class(name='Joseph', age='22')
-        i2 = dict(name='Maria', age=i1)
-        i3 = self.item_class(name='Jesus', age=i2)
+        i1 = self.item_class(name="Joseph", age="22")
+        i2 = dict(name="Maria", age=i1)
+        i3 = self.item_class(name="Jesus", age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
         self.assertEqual(
             exported,
-            {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
+            {
+                "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"},
+                "name": "Jesus",
+            },
         )
-        self.assertEqual(type(exported['age']), dict)
-        self.assertEqual(type(exported['age']['age']), dict)
+        self.assertEqual(type(exported["age"]), dict)
+        self.assertEqual(type(exported["age"]["age"]), dict)
 
     def test_export_list(self):
-        i1 = self.item_class(name='Joseph', age='22')
-        i2 = self.item_class(name='Maria', age=[i1])
-        i3 = self.item_class(name='Jesus', age=[i2])
+        i1 = self.item_class(name="Joseph", age="22")
+        i2 = self.item_class(name="Maria", age=[i1])
+        i3 = self.item_class(name="Jesus", age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
+            {
+                "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}],
+                "name": "Jesus",
+            },
         )
-        self.assertEqual(type(exported['age'][0]), dict)
-        self.assertEqual(type(exported['age'][0]['age'][0]), dict)
+        self.assertEqual(type(exported["age"][0]), dict)
+        self.assertEqual(type(exported["age"][0]["age"][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = self.item_class(name='Joseph', age='22')
-        i2 = dict(name='Maria', age=[i1])
-        i3 = self.item_class(name='Jesus', age=[i2])
+        i1 = self.item_class(name="Joseph", age="22")
+        i2 = dict(name="Maria", age=[i1])
+        i3 = self.item_class(name="Jesus", age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
+            {
+                "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}],
+                "name": "Jesus",
+            },
         )
-        self.assertEqual(type(exported['age'][0]), dict)
-        self.assertEqual(type(exported['age'][0]['age'][0]), dict)
+        self.assertEqual(type(exported["age"][0]), dict)
+        self.assertEqual(type(exported["age"][0]["age"][0]), dict)
 
     def test_export_binary(self):
         with catch_warnings():
-            filterwarnings('ignore', category=ScrapyDeprecationWarning)
+            filterwarnings("ignore", category=ScrapyDeprecationWarning)
             exporter = PythonItemExporter(binary=True)
-            value = self.item_class(name='John\xa3', age='22')
-            expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
+            value = self.item_class(name="John\xa3", age="22")
+            expected = {b"name": b"John\xc2\xa3", b"age": b"22"}
             self.assertEqual(expected, exporter.export_item(value))
 
     def test_nonstring_types_item(self):
@@ -200,7 +216,6 @@ class PythonItemExporterDataclassTest(PythonItemExporterTest):
 
 
 class PprintItemExporterTest(BaseItemExporterTest):
-
     def _get_exporter(self, **kwargs):
         return PprintItemExporter(self.output, **kwargs)
 
@@ -214,7 +229,6 @@ class PprintItemExporterDataclassTest(PprintItemExporterTest):
 
 
 class PickleItemExporterTest(BaseItemExporterTest):
-
     def _get_exporter(self, **kwargs):
         return PickleItemExporter(self.output, **kwargs)
 
@@ -222,8 +236,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(pickle.loads(self.output.getvalue()))
 
     def test_export_multiple_items(self):
-        i1 = self.item_class(name='hello', age='world')
-        i2 = self.item_class(name='bye', age='world')
+        i1 = self.item_class(name="hello", age="world")
+        i2 = self.item_class(name="bye", age="world")
         f = BytesIO()
         ie = PickleItemExporter(f)
         ie.start_exporting()
@@ -250,7 +264,6 @@ class PickleItemExporterDataclassTest(PickleItemExporterTest):
 
 
 class MarshalItemExporterTest(BaseItemExporterTest):
-
     def _get_exporter(self, **kwargs):
         self.output = tempfile.TemporaryFile()
         return MarshalItemExporter(self.output, **kwargs)
@@ -261,7 +274,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
-        item.pop('time')  # datetime is not marshallable
+        item.pop("time")  # datetime is not marshallable
         fp = tempfile.TemporaryFile()
         ie = MarshalItemExporter(fp)
         ie.start_exporting()
@@ -287,11 +300,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
                 sorted(re.split(r"(,|\s+)", line))
                 for line in to_unicode(csv).splitlines(True)
             ]
+
         return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
         self.output.seek(0)
-        self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')
+        self.assertCsvEqual(
+            to_unicode(self.output.read()), "age,name\r\n22,John\xa3\r\n"
+        )
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
@@ -305,21 +321,21 @@ class CsvItemExporterTest(BaseItemExporterTest):
         self.assertExportResult(
             item=self.i,
             fields_to_export=ItemAdapter(self.i).field_names(),
-            expected=b'age,name\r\n22,John\xc2\xa3\r\n',
+            expected=b"age,name\r\n22,John\xc2\xa3\r\n",
         )
 
     def test_header_export_all_dict(self):
         self.assertExportResult(
             item=ItemAdapter(self.i).asdict(),
-            expected=b'age,name\r\n22,John\xc2\xa3\r\n',
+            expected=b"age,name\r\n22,John\xc2\xa3\r\n",
         )
 
     def test_header_export_single_field(self):
         for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
-                fields_to_export=['age'],
-                expected=b'age\r\n22\r\n',
+                fields_to_export=["age"],
+                expected=b"age\r\n22\r\n",
             )
 
     def test_header_export_two_items(self):
@@ -330,15 +346,16 @@ class CsvItemExporterTest(BaseItemExporterTest):
             ie.export_item(item)
             ie.export_item(item)
             ie.finish_exporting()
-            self.assertCsvEqual(output.getvalue(),
-                                b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
+            self.assertCsvEqual(
+                output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n"
+            )
 
     def test_header_no_header_line(self):
         for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 include_headers_line=False,
-                expected=b'22,John\xc2\xa3\r\n',
+                expected=b"22,John\xc2\xa3\r\n",
             )
 
     def test_join_multivalue(self):
@@ -348,14 +365,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
 
         for cls in TestItem2, dict:
             self.assertExportResult(
-                item=cls(name='John', friends=['Mary', 'Paul']),
+                item=cls(name="John", friends=["Mary", "Paul"]),
                 include_headers_line=False,
                 expected='"Mary,Paul",John\r\n',
             )
 
     def test_join_multivalue_not_strings(self):
         self.assertExportResult(
-            item=dict(name='John', friends=[4, 8]),
+            item=dict(name="John", friends=[4, 8]),
             include_headers_line=False,
             expected='"[4, 8]",John\r\n',
         )
@@ -364,24 +381,24 @@ class CsvItemExporterTest(BaseItemExporterTest):
         self.assertExportResult(
             item=self._get_nonstring_types_item(),
             include_headers_line=False,
-            expected='22,False,3.14,2015-01-01 01:01:01\r\n'
+            expected="22,False,3.14,2015-01-01 01:01:01\r\n",
         )
 
     def test_errors_default(self):
         with self.assertRaises(UnicodeEncodeError):
             self.assertExportResult(
-                item=dict(text='W\u0275\u200Brd'),
+                item=dict(text="W\u0275\u200Brd"),
                 expected=None,
-                encoding='windows-1251',
+                encoding="windows-1251",
             )
 
     def test_errors_xmlcharrefreplace(self):
         self.assertExportResult(
-            item=dict(text='W\u0275\u200Brd'),
+            item=dict(text="W\u0275\u200Brd"),
             include_headers_line=False,
-            expected='W&#629;&#8203;rd\r\n',
-            encoding='windows-1251',
-            errors='xmlcharrefreplace',
+            expected="W&#629;&#8203;rd\r\n",
+            encoding="windows-1251",
+            errors="xmlcharrefreplace",
         )
 
 
@@ -391,7 +408,6 @@ class CsvItemExporterDataclassTest(CsvItemExporterTest):
 
 
 class XmlItemExporterTest(BaseItemExporterTest):
-
     def _get_exporter(self, **kwargs):
         return XmlItemExporter(self.output, **kwargs)
 
@@ -405,6 +421,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         def xmlsplit(xmlcontent):
             doc = lxml.etree.fromstring(xmlcontent)
             return xmltuple(doc)
+
         return self.assertEqual(xmlsplit(first), xmlsplit(second), msg)
 
     def assertExportResult(self, item, expected_value):
@@ -418,13 +435,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
     def _check_output(self):
         expected_value = (
             b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+            b"<items><item><age>22</age><name>John\xc2\xa3</name></item></items>"
         )
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
+            self.item_class(name=["John\xa3", "Doe"], age=[1, 2, 3]),
             b"""<?xml version="1.0" encoding="utf-8"?>\n
             <items>
                 <item>
@@ -432,13 +449,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
                     <age><value>1</value><value>2</value><value>3</value></age>
                 </item>
             </items>
-            """
+            """,
         )
 
     def test_nested_item(self):
-        i1 = dict(name='foo\xa3hoo', age='22')
-        i2 = dict(name='bar', age=i1)
-        i3 = self.item_class(name='buz', age=i2)
+        i1 = dict(name="foo\xa3hoo", age="22")
+        i2 = dict(name="bar", age=i1)
+        i3 = self.item_class(name="buz", age=i2)
 
         self.assertExportResult(
             i3,
@@ -455,13 +472,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
                         <name>buz</name>
                     </item>
                 </items>
-            """
+            """,
         )
 
     def test_nested_list_item(self):
-        i1 = dict(name='foo')
-        i2 = dict(name='bar', v2={"egg": ["spam"]})
-        i3 = self.item_class(name='buz', age=[i1, i2])
+        i1 = dict(name="foo")
+        i2 = dict(name="bar", v2={"egg": ["spam"]})
+        i3 = self.item_class(name="buz", age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -475,7 +492,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
                         <name>buz</name>
                     </item>
                 </items>
-            """
+            """,
         )
 
     def test_nonstring_types_item(self):
@@ -491,7 +508,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
                        <time>2015-01-01 01:01:01</time>
                    </item>
                 </items>
-            """
+            """,
         )
 
 
@@ -503,7 +520,10 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
-    _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
+    _expected_nested = {
+        "name": "Jesus",
+        "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}},
+    }
 
     def _get_exporter(self, **kwargs):
         return JsonLinesItemExporter(self.output, **kwargs)
@@ -513,9 +533,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name='Joseph', age='22')
-        i2 = dict(name='Maria', age=i1)
-        i3 = self.item_class(name='Jesus', age=i2)
+        i1 = self.item_class(name="Joseph", age="22")
+        i2 = dict(name="Maria", age=i1)
+        i3 = self.item_class(name="Jesus", age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -534,7 +554,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        item['time'] = str(item['time'])
+        item["time"] = str(item["time"])
         self.assertEqual(exported, item)
 
 
@@ -561,7 +581,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
+        self.assertEqual(
+            exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]
+        )
 
     def test_two_items(self):
         self.assertTwoItemsExported(self.i)
@@ -570,25 +592,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name='Joseph\xa3', age='22')
-        i2 = self.item_class(name='Maria', age=i1)
-        i3 = self.item_class(name='Jesus', age=i2)
+        i1 = self.item_class(name="Joseph\xa3", age="22")
+        i2 = self.item_class(name="Maria", age=i1)
+        i3 = self.item_class(name="Jesus", age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
+        expected = {
+            "name": "Jesus",
+            "age": {"name": "Maria", "age": ItemAdapter(i1).asdict()},
+        }
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
-        i1 = dict(name='Joseph\xa3', age='22')
-        i2 = self.item_class(name='Maria', age=i1)
-        i3 = dict(name='Jesus', age=i2)
+        i1 = dict(name="Joseph\xa3", age="22")
+        i2 = self.item_class(name="Maria", age=i1)
+        i3 = dict(name="Jesus", age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
+        expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}}
         self.assertEqual(exported, [expected])
 
     def test_nonstring_types_item(self):
@@ -597,7 +622,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        item['time'] = str(item['time'])
+        item["time"] = str(item["time"])
         self.assertEqual(exported, [item])
 
 
@@ -618,20 +643,24 @@ class CustomExporterItemTest(unittest.TestCase):
     def test_exporter_custom_serializer(self):
         class CustomItemExporter(BaseItemExporter):
             def serialize_field(self, field, name, value):
-                if name == 'age':
+                if name == "age":
                     return str(int(value) + 1)
                 return super().serialize_field(field, name, value)
 
-        i = self.item_class(name='John', age='22')
+        i = self.item_class(name="John", age="22")
         a = ItemAdapter(i)
         ie = CustomItemExporter()
 
-        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
-        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
+        self.assertEqual(
+            ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John"
+        )
+        self.assertEqual(
+            ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "23"
+        )
 
-        i2 = {'name': 'John', 'age': '22'}
-        self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
-        self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
+        i2 = {"name": "John", "age": "22"}
+        self.assertEqual(ie.serialize_field({}, "name", i2["name"]), "John")
+        self.assertEqual(ie.serialize_field({}, "age", i2["age"]), "23")
 
 
 class CustomExporterDataclassTest(CustomExporterItemTest):
@@ -639,5 +668,5 @@ class CustomExporterDataclassTest(CustomExporterItemTest):
     item_class = TestDataClass
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 1e716b94a..e36c45d8e 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -24,7 +24,7 @@ class TelnetExtensionTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_bad_credentials(self):
         console, portal = self._get_console_and_portal()
-        creds = credentials.UsernamePassword(b'username', b'password')
+        creds = credentials.UsernamePassword(b"username", b"password")
         d = portal.login(creds, None, ITelnetProtocol)
         yield self.assertFailure(d, ValueError)
         console.stop_listening()
@@ -33,8 +33,7 @@ class TelnetExtensionTest(unittest.TestCase):
     def test_good_credentials(self):
         console, portal = self._get_console_and_portal()
         creds = credentials.UsernamePassword(
-            console.username.encode('utf8'),
-            console.password.encode('utf8')
+            console.username.encode("utf8"), console.password.encode("utf8")
         )
         d = portal.login(creds, None, ITelnetProtocol)
         yield d
@@ -43,11 +42,11 @@ class TelnetExtensionTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_custom_credentials(self):
         settings = {
-            'TELNETCONSOLE_USERNAME': 'user',
-            'TELNETCONSOLE_PASSWORD': 'pass',
+            "TELNETCONSOLE_USERNAME": "user",
+            "TELNETCONSOLE_PASSWORD": "pass",
         }
         console, portal = self._get_console_and_portal(settings=settings)
-        creds = credentials.UsernamePassword(b'user', b'pass')
+        creds = credentials.UsernamePassword(b"user", b"pass")
         d = portal.login(creds, None, ITelnetProtocol)
         yield d
         console.stop_listening()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 97c3a74b3..890c88c7e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -57,29 +57,28 @@ from tests.spiders import ItemSpider
 
 
 def path_to_url(path):
-    return urljoin('file:', pathname2url(str(path)))
+    return urljoin("file:", pathname2url(str(path)))
 
 
 def printf_escape(string):
-    return string.replace('%', '%%')
+    return string.replace("%", "%%")
 
 
 def build_url(path: Union[str, PathLike]) -> str:
     path_str = str(path)
-    if path_str[0] != '/':
-        path_str = '/' + path_str
-    return urljoin('file:', path_str)
+    if path_str[0] != "/":
+        path_str = "/" + path_str
+    return urljoin("file:", path_str)
 
 
 class FileFeedStorageTest(unittest.TestCase):
-
     def test_store_file_uri(self):
         path = Path(self.mktemp()).resolve()
         uri = path_to_file_uri(str(path))
         return self._assert_stores(FileFeedStorage(uri), path)
 
     def test_store_file_uri_makedirs(self):
-        path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt'
+        path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt"
         uri = path_to_file_uri(str(path))
         return self._assert_stores(FileFeedStorage(uri), path)
 
@@ -112,8 +111,7 @@ class FileFeedStorageTest(unittest.TestCase):
     def test_overwrite(self):
         path = self._store({"overwrite": True})
         return self._assert_stores(
-            FileFeedStorage(str(path), feed_options={"overwrite": True}),
-            path
+            FileFeedStorage(str(path), feed_options={"overwrite": True}), path
         )
 
     @defer.inlineCallbacks
@@ -130,10 +128,9 @@ class FileFeedStorageTest(unittest.TestCase):
 
 
 class FTPFeedStorageTest(unittest.TestCase):
-
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
-            name = 'test_spider'
+            name = "test_spider"
 
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
@@ -162,9 +159,9 @@ class FTPFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_append(self):
         with MockFTPServer() as ftp_server:
-            filename = 'file'
+            filename = "file"
             url = ftp_server.url(filename)
-            feed_options = {'overwrite': False}
+            feed_options = {"overwrite": False}
             yield self._store(url, b"foo", feed_options=feed_options)
             yield self._store(url, b"bar", feed_options=feed_options)
             self._assert_stored(ftp_server.path / filename, b"foobar")
@@ -172,7 +169,7 @@ class FTPFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_overwrite(self):
         with MockFTPServer() as ftp_server:
-            filename = 'file'
+            filename = "file"
             url = ftp_server.url(filename)
             yield self._store(url, b"foo")
             yield self._store(url, b"bar")
@@ -181,10 +178,10 @@ class FTPFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_append_active_mode(self):
         with MockFTPServer() as ftp_server:
-            settings = {'FEED_STORAGE_FTP_ACTIVE': True}
-            filename = 'file'
+            settings = {"FEED_STORAGE_FTP_ACTIVE": True}
+            filename = "file"
             url = ftp_server.url(filename)
-            feed_options = {'overwrite': False}
+            feed_options = {"overwrite": False}
             yield self._store(url, b"foo", feed_options=feed_options, settings=settings)
             yield self._store(url, b"bar", feed_options=feed_options, settings=settings)
             self._assert_stored(ftp_server.path / filename, b"foobar")
@@ -192,8 +189,8 @@ class FTPFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_overwrite_active_mode(self):
         with MockFTPServer() as ftp_server:
-            settings = {'FEED_STORAGE_FTP_ACTIVE': True}
-            filename = 'file'
+            settings = {"FEED_STORAGE_FTP_ACTIVE": True}
+            filename = "file"
             url = ftp_server.url(filename)
             yield self._store(url, b"foo", settings=settings)
             yield self._store(url, b"bar", settings=settings)
@@ -201,16 +198,15 @@ class FTPFeedStorageTest(unittest.TestCase):
 
     def test_uri_auth_quote(self):
         # RFC3986: 3.2.1. User Information
-        pw_quoted = quote(string.punctuation, safe='')
-        st = FTPFeedStorage(f'ftp://foo:{pw_quoted}@example.com/some_path', {})
+        pw_quoted = quote(string.punctuation, safe="")
+        st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {})
         self.assertEqual(st.password, string.punctuation)
 
 
 class BlockingFeedStorageTest(unittest.TestCase):
-
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
-            name = 'test_spider'
+            name = "test_spider"
 
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
@@ -227,7 +223,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
         b = BlockingFeedStorage()
 
         tests_path = Path(__file__).resolve().parent
-        spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)})
+        spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)})
         tmp = b.open(spider)
         tmp_path = Path(tmp.name).parent
         self.assertEqual(tmp_path, tests_path)
@@ -236,66 +232,72 @@ class BlockingFeedStorageTest(unittest.TestCase):
         b = BlockingFeedStorage()
 
         tests_path = Path(__file__).resolve().parent
-        invalid_path = tests_path / 'invalid_path'
-        spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)})
+        invalid_path = tests_path / "invalid_path"
+        spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
 
         self.assertRaises(OSError, b.open, spider=spider)
 
 
 class S3FeedStorageTest(unittest.TestCase):
-
     def test_parse_credentials(self):
         skip_if_no_boto()
-        aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
-                           'AWS_SECRET_ACCESS_KEY': 'settings_secret',
-                           'AWS_SESSION_TOKEN': 'settings_token'}
+        aws_credentials = {
+            "AWS_ACCESS_KEY_ID": "settings_key",
+            "AWS_SECRET_ACCESS_KEY": "settings_secret",
+            "AWS_SESSION_TOKEN": "settings_token",
+        }
         crawler = get_crawler(settings_dict=aws_credentials)
         # Instantiate with crawler
         storage = S3FeedStorage.from_crawler(
             crawler,
-            's3://mybucket/export.csv',
+            "s3://mybucket/export.csv",
         )
-        self.assertEqual(storage.access_key, 'settings_key')
-        self.assertEqual(storage.secret_key, 'settings_secret')
-        self.assertEqual(storage.session_token, 'settings_token')
+        self.assertEqual(storage.access_key, "settings_key")
+        self.assertEqual(storage.secret_key, "settings_secret")
+        self.assertEqual(storage.session_token, "settings_token")
         # Instantiate directly
-        storage = S3FeedStorage('s3://mybucket/export.csv',
-                                aws_credentials['AWS_ACCESS_KEY_ID'],
-                                aws_credentials['AWS_SECRET_ACCESS_KEY'],
-                                session_token=aws_credentials['AWS_SESSION_TOKEN'])
-        self.assertEqual(storage.access_key, 'settings_key')
-        self.assertEqual(storage.secret_key, 'settings_secret')
-        self.assertEqual(storage.session_token, 'settings_token')
+        storage = S3FeedStorage(
+            "s3://mybucket/export.csv",
+            aws_credentials["AWS_ACCESS_KEY_ID"],
+            aws_credentials["AWS_SECRET_ACCESS_KEY"],
+            session_token=aws_credentials["AWS_SESSION_TOKEN"],
+        )
+        self.assertEqual(storage.access_key, "settings_key")
+        self.assertEqual(storage.secret_key, "settings_secret")
+        self.assertEqual(storage.session_token, "settings_token")
         # URI priority > settings priority
-        storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv',
-                                aws_credentials['AWS_ACCESS_KEY_ID'],
-                                aws_credentials['AWS_SECRET_ACCESS_KEY'])
-        self.assertEqual(storage.access_key, 'uri_key')
-        self.assertEqual(storage.secret_key, 'uri_secret')
+        storage = S3FeedStorage(
+            "s3://uri_key:uri_secret@mybucket/export.csv",
+            aws_credentials["AWS_ACCESS_KEY_ID"],
+            aws_credentials["AWS_SECRET_ACCESS_KEY"],
+        )
+        self.assertEqual(storage.access_key, "uri_key")
+        self.assertEqual(storage.secret_key, "uri_secret")
 
     @defer.inlineCallbacks
     def test_store(self):
         skip_if_no_boto()
 
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
         }
         crawler = get_crawler(settings_dict=settings)
-        bucket = 'mybucket'
-        key = 'export.csv'
-        storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}')
+        bucket = "mybucket"
+        key = "export.csv"
+        storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
         verifyObject(IFeedStorage, storage)
 
         file = mock.MagicMock()
         from botocore.stub import Stubber
+
         with Stubber(storage.s3_client) as stub:
             stub.add_response(
-                'put_object',
+                "put_object",
                 expected_params={
-                    'Body': file,
-                    'Bucket': bucket,
-                    'Key': key,
+                    "Body": file,
+                    "Bucket": bucket,
+                    "Key": key,
                 },
                 service_response={},
             )
@@ -309,172 +311,154 @@ class S3FeedStorageTest(unittest.TestCase):
                     mock.call.seek(0),
                     # The call to read does not happen with Stubber
                     mock.call.close(),
-                ]
+                ],
             )
 
     def test_init_without_acl(self):
-        storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key'
-        )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
+        storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.acl, None)
 
     def test_init_with_acl(self):
         storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key',
-            'custom-acl'
+            "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.acl, 'custom-acl')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.acl, "custom-acl")
 
     def test_init_with_endpoint_url(self):
         storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key',
-            endpoint_url='https://example.com'
+            "s3://mybucket/export.csv",
+            "access_key",
+            "secret_key",
+            endpoint_url="https://example.com",
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.endpoint_url, 'https://example.com')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.endpoint_url, "https://example.com")
 
     def test_from_crawler_without_acl(self):
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
         }
         crawler = get_crawler(settings_dict=settings)
         storage = S3FeedStorage.from_crawler(
             crawler,
-            's3://mybucket/export.csv',
+            "s3://mybucket/export.csv",
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.acl, None)
 
     def test_without_endpoint_url(self):
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
         }
         crawler = get_crawler(settings_dict=settings)
         storage = S3FeedStorage.from_crawler(
             crawler,
-            's3://mybucket/export.csv',
+            "s3://mybucket/export.csv",
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.endpoint_url, None)
 
     def test_from_crawler_with_acl(self):
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
-            'FEED_STORAGE_S3_ACL': 'custom-acl',
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
+            "FEED_STORAGE_S3_ACL": "custom-acl",
         }
         crawler = get_crawler(settings_dict=settings)
         storage = S3FeedStorage.from_crawler(
             crawler,
-            's3://mybucket/export.csv',
+            "s3://mybucket/export.csv",
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.acl, 'custom-acl')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.acl, "custom-acl")
 
     def test_from_crawler_with_endpoint_url(self):
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
-            'AWS_ENDPOINT_URL': 'https://example.com',
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
+            "AWS_ENDPOINT_URL": "https://example.com",
         }
         crawler = get_crawler(settings_dict=settings)
-        storage = S3FeedStorage.from_crawler(
-            crawler,
-            's3://mybucket/export.csv'
-        )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.endpoint_url, 'https://example.com')
+        storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.endpoint_url, "https://example.com")
 
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
         skip_if_no_boto()
         storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key',
+            "s3://mybucket/export.csv",
+            "access_key",
+            "secret_key",
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
         self.assertEqual(storage.acl, None)
 
         storage.s3_client = mock.MagicMock()
-        yield storage.store(BytesIO(b'test file'))
-        self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1])
+        yield storage.store(BytesIO(b"test file"))
+        self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1])
 
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
         skip_if_no_boto()
         storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key',
-            'custom-acl'
+            "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
         )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.acl, 'custom-acl')
+        self.assertEqual(storage.access_key, "access_key")
+        self.assertEqual(storage.secret_key, "secret_key")
+        self.assertEqual(storage.acl, "custom-acl")
 
         storage.s3_client = mock.MagicMock()
-        yield storage.store(BytesIO(b'test file'))
+        yield storage.store(BytesIO(b"test file"))
         self.assertEqual(
-            storage.s3_client.put_object.call_args[1].get('ACL'),
-            'custom-acl'
+            storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl"
         )
 
     def test_overwrite_default(self):
         with LogCapture() as log:
             S3FeedStorage(
-                's3://mybucket/export.csv',
-                'access_key',
-                'secret_key',
-                'custom-acl'
+                "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
             )
-        self.assertNotIn('S3 does not support appending to files', str(log))
+        self.assertNotIn("S3 does not support appending to files", str(log))
 
     def test_overwrite_false(self):
         with LogCapture() as log:
             S3FeedStorage(
-                's3://mybucket/export.csv',
-                'access_key',
-                'secret_key',
-                'custom-acl',
-                feed_options={'overwrite': False},
+                "s3://mybucket/export.csv",
+                "access_key",
+                "secret_key",
+                "custom-acl",
+                feed_options={"overwrite": False},
             )
-        self.assertIn('S3 does not support appending to files', str(log))
+        self.assertIn("S3 does not support appending to files", str(log))
 
 
 class GCSFeedStorageTest(unittest.TestCase):
-
     def test_parse_settings(self):
         try:
             from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
+        settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
         crawler = get_crawler(settings_dict=settings)
-        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
-        assert storage.project_id == '123'
-        assert storage.acl == 'publicRead'
-        assert storage.bucket_name == 'mybucket'
-        assert storage.blob_name == 'export.csv'
+        storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
+        assert storage.project_id == "123"
+        assert storage.acl == "publicRead"
+        assert storage.bucket_name == "mybucket"
+        assert storage.blob_name == "export.csv"
 
     def test_parse_empty_acl(self):
         try:
@@ -482,14 +466,14 @@ class GCSFeedStorageTest(unittest.TestCase):
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
+        settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
         crawler = get_crawler(settings_dict=settings)
-        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
         assert storage.acl is None
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
+        settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None}
         crawler = get_crawler(settings_dict=settings)
-        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
         assert storage.acl is None
 
     @defer.inlineCallbacks
@@ -499,11 +483,11 @@ class GCSFeedStorageTest(unittest.TestCase):
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        uri = 'gs://mybucket/export.csv'
-        project_id = 'myproject-123'
-        acl = 'publicRead'
+        uri = "gs://mybucket/export.csv"
+        project_id = "myproject-123"
+        acl = "publicRead"
         (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
-        with mock.patch('google.cloud.storage.Client') as m:
+        with mock.patch("google.cloud.storage.Client") as m:
             m.return_value = client_mock
 
             f = mock.Mock()
@@ -512,17 +496,16 @@ class GCSFeedStorageTest(unittest.TestCase):
 
             f.seek.assert_called_once_with(0)
             m.assert_called_once_with(project=project_id)
-            client_mock.get_bucket.assert_called_once_with('mybucket')
-            bucket_mock.blob.assert_called_once_with('export.csv')
+            client_mock.get_bucket.assert_called_once_with("mybucket")
+            bucket_mock.blob.assert_called_once_with("export.csv")
             blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
 
 
 class StdoutFeedStorageTest(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_store(self):
         out = BytesIO()
-        storage = StdoutFeedStorage('stdout:', _stdout=out)
+        storage = StdoutFeedStorage("stdout:", _stdout=out)
         file = storage.open(scrapy.Spider("default"))
         file.write(b"content")
         yield storage.store(file)
@@ -530,13 +513,17 @@ class StdoutFeedStorageTest(unittest.TestCase):
 
     def test_overwrite_default(self):
         with LogCapture() as log:
-            StdoutFeedStorage('stdout:')
-        self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log))
+            StdoutFeedStorage("stdout:")
+        self.assertNotIn(
+            "Standard output (stdout) storage does not support overwriting", str(log)
+        )
 
     def test_overwrite_true(self):
         with LogCapture() as log:
-            StdoutFeedStorage('stdout:', feed_options={'overwrite': True})
-        self.assertIn('Standard output (stdout) storage does not support overwriting', str(log))
+            StdoutFeedStorage("stdout:", feed_options={"overwrite": True})
+        self.assertIn(
+            "Standard output (stdout) storage does not support overwriting", str(log)
+        )
 
 
 class FromCrawlerMixin:
@@ -553,7 +540,6 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin):
 
 
 class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
-
     @classmethod
     def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
         cls.init_with_crawler = True
@@ -561,7 +547,6 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
 
 
 class DummyBlockingFeedStorage(BlockingFeedStorage):
-
     def __init__(self, uri, *args, feed_options=None):
         self.path = Path(file_uri_to_path(uri))
 
@@ -569,16 +554,15 @@ class DummyBlockingFeedStorage(BlockingFeedStorage):
         dirname = self.path.parent
         if dirname and not dirname.exists():
             dirname.mkdir(parents=True)
-        with self.path.open('ab') as output_file:
+        with self.path.open("ab") as output_file:
             output_file.write(file.read())
 
         file.close()
 
 
 class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
-
     def _store_in_thread(self, file):
-        raise OSError('Cannot store')
+        raise OSError("Cannot store")
 
 
 @implementer(IFeedStorage)
@@ -593,10 +577,10 @@ class LogOnStoreFileStorage:
         self.logger = getLogger()
 
     def open(self, spider):
-        return tempfile.NamedTemporaryFile(prefix='feed-')
+        return tempfile.NamedTemporaryFile(prefix="feed-")
 
     def store(self, file):
-        self.logger.info('Storage.store is called')
+        self.logger.info("Storage.store is called")
         file.close()
 
 
@@ -612,9 +596,9 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         foo = scrapy.Field()
         hello = scrapy.Field()
 
-    def _random_temp_filename(self, inter_dir='') -> Path:
+    def _random_temp_filename(self, inter_dir="") -> Path:
         chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = ''.join(chars)
+        filename = "".join(chars)
         return Path(self.temp_dir, inter_dir, filename)
 
     def setUp(self):
@@ -630,7 +614,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         """
 
         class TestSpider(scrapy.Spider):
-            name = 'testspider'
+            name = "testspider"
 
             def parse(self, response):
                 for item in items:
@@ -646,7 +630,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         """
 
         class TestSpider(scrapy.Spider):
-            name = 'testspider'
+            name = "testspider"
 
             def parse(self, response):
                 pass
@@ -685,10 +669,10 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
-        """ Run spider with specified settings; return exported data. """
+        """Run spider with specified settings; return exported data."""
 
-        FEEDS = settings.get('FEEDS') or {}
-        settings['FEEDS'] = {
+        FEEDS = settings.get("FEEDS") or {}
+        settings["FEEDS"] = {
             printf_escape(path_to_url(file_path)): feed_options
             for file_path, feed_options in FEEDS.items()
         }
@@ -696,7 +680,7 @@ class FeedExportTest(FeedExportTestBase):
         content = {}
         try:
             with MockServer() as s:
-                spider_cls.start_urls = [s.url('/')]
+                spider_cls.start_urls = [s.url("/")]
                 crawler = get_crawler(spider_cls, settings)
                 yield crawler.crawl()
 
@@ -704,7 +688,7 @@ class FeedExportTest(FeedExportTestBase):
                 if not Path(file_path).exists():
                     continue
 
-                content[feed_options['format']] = Path(file_path).read_bytes()
+                content[feed_options["format"]] = Path(file_path).read_bytes()
 
         finally:
             for file_path in FEEDS.keys():
@@ -718,88 +702,102 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'csv'},
-            },
-        })
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": "csv"},
+                },
+            }
+        )
         data = yield self.exported_data(items, settings)
-        reader = csv.DictReader(to_unicode(data['csv']).splitlines())
+        reader = csv.DictReader(to_unicode(data["csv"]).splitlines())
         self.assertEqual(reader.fieldnames, list(header))
         self.assertEqual(rows, list(reader))
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'jl'},
-            },
-        })
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": "jl"},
+                },
+            }
+        )
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
     @defer.inlineCallbacks
     def assertExportedXml(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'xml'},
-            },
-        })
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": "xml"},
+                },
+            }
+        )
         data = yield self.exported_data(items, settings)
         rows = [{k: v for k, v in row.items() if v} for row in rows]
-        root = lxml.etree.fromstring(data['xml'])
-        got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]
+        root = lxml.etree.fromstring(data["xml"])
+        got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")]
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
     def assertExportedMultiple(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'xml'},
-                self._random_temp_filename(): {'format': 'json'},
-            },
-        })
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": "xml"},
+                    self._random_temp_filename(): {"format": "json"},
+                },
+            }
+        )
         data = yield self.exported_data(items, settings)
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         # XML
-        root = lxml.etree.fromstring(data['xml'])
-        xml_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]
+        root = lxml.etree.fromstring(data["xml"])
+        xml_rows = [{e.tag: e.text for e in it} for it in root.findall("item")]
         self.assertEqual(rows, xml_rows)
         # JSON
-        json_rows = json.loads(to_unicode(data['json']))
+        json_rows = json.loads(to_unicode(data["json"]))
         self.assertEqual(rows, json_rows)
 
     @defer.inlineCallbacks
     def assertExportedPickle(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'pickle'},
-            },
-        })
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": "pickle"},
+                },
+            }
+        )
         data = yield self.exported_data(items, settings)
         expected = [{k: v for k, v in row.items() if v} for row in rows]
         import pickle
-        result = self._load_until_eof(data['pickle'], load_func=pickle.load)
+
+        result = self._load_until_eof(data["pickle"], load_func=pickle.load)
         self.assertEqual(expected, result)
 
     @defer.inlineCallbacks
     def assertExportedMarshal(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'marshal'},
-            },
-        })
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": "marshal"},
+                },
+            }
+        )
         data = yield self.exported_data(items, settings)
         expected = [{k: v for k, v in row.items() if v} for row in rows]
         import marshal
-        result = self._load_until_eof(data['marshal'], load_func=marshal.load)
+
+        result = self._load_until_eof(data["marshal"], load_func=marshal.load)
         self.assertEqual(expected, result)
 
     @defer.inlineCallbacks
@@ -814,8 +812,12 @@ class FeedExportTest(FeedExportTestBase):
         crawler = get_crawler(ItemSpider, settings)
         with MockServer() as mockserver:
             yield crawler.crawl(mockserver=mockserver)
-        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
-        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
+        self.assertIn(
+            "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()
+        )
+        self.assertEqual(
+            crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1
+        )
 
     @defer.inlineCallbacks
     def test_stats_file_failed(self):
@@ -832,17 +834,22 @@ class FeedExportTest(FeedExportTestBase):
             stack.enter_context(
                 mock.patch(
                     "scrapy.extensions.feedexport.FileFeedStorage.store",
-                    side_effect=KeyError("foo"))
+                    side_effect=KeyError("foo"),
+                )
             )
             yield crawler.crawl(mockserver=mockserver)
-        self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats())
-        self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1)
+        self.assertIn(
+            "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats()
+        )
+        self.assertEqual(
+            crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1
+        )
 
     @defer.inlineCallbacks
     def test_stats_multiple_file(self):
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
             "FEEDS": {
                 printf_escape(path_to_url(str(self._random_temp_filename()))): {
                     "format": "json",
@@ -852,101 +859,113 @@ class FeedExportTest(FeedExportTestBase):
                 },
                 "stdout:": {
                     "format": "xml",
-                }
+                },
             },
         }
         crawler = get_crawler(ItemSpider, settings)
         with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"):
             yield crawler.crawl(mockserver=mockserver)
-        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
-        self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats())
-        self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats())
-        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
-        self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1)
-        self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1)
+        self.assertIn(
+            "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()
+        )
+        self.assertIn(
+            "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()
+        )
+        self.assertIn(
+            "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()
+        )
+        self.assertEqual(
+            crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1
+        )
+        self.assertEqual(
+            crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1
+        )
+        self.assertEqual(
+            crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1
+        )
 
     @defer.inlineCallbacks
     def test_export_items(self):
         # feed exporters use field names from Item
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
         ]
         rows = [
-            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
-            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
+            {"egg": "spam1", "foo": "bar1", "baz": ""},
+            {"egg": "spam2", "foo": "bar2", "baz": "quux2"},
         ]
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
-        for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+        for fmt in ("json", "jsonlines", "xml", "csv"):
             settings = {
-                'FEEDS': {
-                    self._random_temp_filename(): {'format': fmt},
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": fmt},
                 },
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(b'', data[fmt])
+            self.assertEqual(b"", data[fmt])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
         formats = (
-            ('json', b'[]'),
-            ('jsonlines', b''),
-            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
-            ('csv', b''),
+            ("json", b"[]"),
+            ("jsonlines", b""),
+            ("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ("csv", b""),
         )
 
         for fmt, expctd in formats:
             settings = {
-                'FEEDS': {
-                    self._random_temp_filename(): {'format': fmt},
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": fmt},
                 },
-                'FEED_STORE_EMPTY': True,
-                'FEED_EXPORT_INDENT': None,
+                "FEED_STORE_EMPTY": True,
+                "FEED_EXPORT_INDENT": None,
             }
             data = yield self.exported_no_data(settings)
             self.assertEqual(expctd, data[fmt])
 
     @defer.inlineCallbacks
     def test_export_no_items_multiple_feeds(self):
-        """ Make sure that `storage.store` is called for every feed. """
+        """Make sure that `storage.store` is called for every feed."""
         settings = {
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'json'},
-                self._random_temp_filename(): {'format': 'xml'},
-                self._random_temp_filename(): {'format': 'csv'},
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+                self._random_temp_filename(): {"format": "xml"},
+                self._random_temp_filename(): {"format": "csv"},
             },
-            'FEED_STORAGES': {'file': LogOnStoreFileStorage},
-            'FEED_STORE_EMPTY': False
+            "FEED_STORAGES": {"file": LogOnStoreFileStorage},
+            "FEED_STORE_EMPTY": False,
         }
 
         with LogCapture() as log:
             yield self.exported_no_data(settings)
 
         print(log)
-        self.assertEqual(str(log).count('Storage.store is called'), 3)
+        self.assertEqual(str(log).count("Storage.store is called"), 3)
 
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
 
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
-            self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),
-            {'hello': 'world4', 'egg': 'spam4'},
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem2({"hello": "world2", "foo": "bar2"}),
+            self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
+            {"hello": "world4", "egg": "spam4"},
         ]
 
         # by default, Scrapy uses fields of the first Item for CSV and
         # all fields for JSON Lines
         header = self.MyItem.fields.keys()
         rows_csv = [
-            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
-            {'egg': '', 'foo': 'bar2', 'baz': ''},
-            {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
-            {'egg': 'spam4', 'foo': '', 'baz': ''},
+            {"egg": "spam1", "foo": "bar1", "baz": ""},
+            {"egg": "", "foo": "bar2", "baz": ""},
+            {"egg": "spam3", "foo": "bar3", "baz": "quux3"},
+            {"egg": "spam4", "foo": "", "baz": ""},
         ]
         rows_jl = [dict(row) for row in items]
         yield self.assertExportedCsv(items, header, rows_csv)
@@ -955,106 +974,103 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_items_empty_field_list(self):
         # FEED_EXPORT_FIELDS==[] means the same as default None
-        items = [{'foo': 'bar'}]
+        items = [{"foo": "bar"}]
         header = ["foo"]
-        rows = [{'foo': 'bar'}]
-        settings = {'FEED_EXPORT_FIELDS': []}
+        rows = [{"foo": "bar"}]
+        settings = {"FEED_EXPORT_FIELDS": []}
         yield self.assertExportedCsv(items, header, rows)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
     def test_export_items_field_list(self):
-        items = [{'foo': 'bar'}]
+        items = [{"foo": "bar"}]
         header = ["foo", "baz"]
-        rows = [{'foo': 'bar', 'baz': ''}]
-        settings = {'FEED_EXPORT_FIELDS': header}
+        rows = [{"foo": "bar", "baz": ""}]
+        settings = {"FEED_EXPORT_FIELDS": header}
         yield self.assertExported(items, header, rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_items_comma_separated_field_list(self):
-        items = [{'foo': 'bar'}]
+        items = [{"foo": "bar"}]
         header = ["foo", "baz"]
-        rows = [{'foo': 'bar', 'baz': ''}]
-        settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+        rows = [{"foo": "bar", "baz": ""}]
+        settings = {"FEED_EXPORT_FIELDS": ",".join(header)}
         yield self.assertExported(items, header, rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_items_json_field_list(self):
-        items = [{'foo': 'bar'}]
+        items = [{"foo": "bar"}]
         header = ["foo", "baz"]
-        rows = [{'foo': 'bar', 'baz': ''}]
-        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        rows = [{"foo": "bar", "baz": ""}]
+        settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
         yield self.assertExported(items, header, rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_items_field_names(self):
-        items = [{'foo': 'bar'}]
-        header = {'foo': 'Foo'}
-        rows = [{'Foo': 'bar'}]
-        settings = {'FEED_EXPORT_FIELDS': header}
-        yield self.assertExported(items, list(header.values()), rows,
-                                  settings=settings)
+        items = [{"foo": "bar"}]
+        header = {"foo": "Foo"}
+        rows = [{"Foo": "bar"}]
+        settings = {"FEED_EXPORT_FIELDS": header}
+        yield self.assertExported(items, list(header.values()), rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_items_dict_field_names(self):
-        items = [{'foo': 'bar'}]
+        items = [{"foo": "bar"}]
         header = {
-            'baz': 'Baz',
-            'foo': 'Foo',
+            "baz": "Baz",
+            "foo": "Foo",
         }
-        rows = [{'Baz': '', 'Foo': 'bar'}]
-        settings = {'FEED_EXPORT_FIELDS': header}
-        yield self.assertExported(items, ['Baz', 'Foo'], rows,
-                                  settings=settings)
+        rows = [{"Baz": "", "Foo": "bar"}]
+        settings = {"FEED_EXPORT_FIELDS": header}
+        yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_items_json_field_names(self):
-        items = [{'foo': 'bar'}]
-        header = {'foo': 'Foo'}
-        rows = [{'Foo': 'bar'}]
-        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
-        yield self.assertExported(items, list(header.values()), rows,
-                                  settings=settings)
+        items = [{"foo": "bar"}]
+        header = {"foo": "Foo"}
+        rows = [{"Foo": "bar"}]
+        settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
+        yield self.assertExported(items, list(header.values()), rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_based_on_item_classes(self):
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
-            {'hello': 'world3', 'egg': 'spam3'},
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem2({"hello": "world2", "foo": "bar2"}),
+            {"hello": "world3", "egg": "spam3"},
         ]
 
         formats = {
-            'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',
-            'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',
-            'jsonlines': (
+            "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n",
+            "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]',
+            "jsonlines": (
                 b'{"foo": "bar1", "egg": "spam1"}\n'
                 b'{"hello": "world2", "foo": "bar2"}\n'
             ),
-            'xml': (
+            "xml": (
                 b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
-                b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
-                b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'
-                b'</hello><egg>spam3</egg></item>\n</items>'
+                b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
+                b"world2</hello><foo>bar2</foo></item>\n<item><hello>world3"
+                b"</hello><egg>spam3</egg></item>\n</items>"
             ),
         }
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 self._random_temp_filename(): {
-                    'format': 'csv',
-                    'item_classes': [self.MyItem],
+                    "format": "csv",
+                    "item_classes": [self.MyItem],
                 },
                 self._random_temp_filename(): {
-                    'format': 'json',
-                    'item_classes': [self.MyItem2],
+                    "format": "json",
+                    "item_classes": [self.MyItem2],
                 },
                 self._random_temp_filename(): {
-                    'format': 'jsonlines',
-                    'item_classes': [self.MyItem, self.MyItem2],
+                    "format": "jsonlines",
+                    "item_classes": [self.MyItem, self.MyItem2],
                 },
                 self._random_temp_filename(): {
-                    'format': 'xml',
+                    "format": "xml",
                 },
             },
         }
@@ -1066,9 +1082,9 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_based_on_custom_filters(self):
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
-            {'hello': 'world3', 'egg': 'spam3'},
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem2({"hello": "world2", "foo": "bar2"}),
+            {"hello": "world3", "egg": "spam3"},
         ]
 
         MyItem = self.MyItem
@@ -1082,40 +1098,40 @@ class FeedExportTest(FeedExportTestBase):
 
         class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
             def accepts(self, item):
-                if 'foo' not in item.fields:
+                if "foo" not in item.fields:
                     return False
                 return True
 
         class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
             def accepts(self, item):
-                if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":
+                if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1":
                     return True
                 return False
 
         formats = {
-            'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
-            'xml': (
+            "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
+            "xml": (
                 b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
-                b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
-                b'world2</hello><foo>bar2</foo></item>\n</items>'
+                b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
+                b"world2</hello><foo>bar2</foo></item>\n</items>"
             ),
-            'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',
+            "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n',
         }
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 self._random_temp_filename(): {
-                    'format': 'json',
-                    'item_filter': CustomFilter1,
+                    "format": "json",
+                    "item_filter": CustomFilter1,
                 },
                 self._random_temp_filename(): {
-                    'format': 'xml',
-                    'item_filter': CustomFilter2,
+                    "format": "xml",
+                    "item_filter": CustomFilter2,
                 },
                 self._random_temp_filename(): {
-                    'format': 'jsonlines',
-                    'item_classes': [self.MyItem, self.MyItem2],
-                    'item_filter': CustomFilter3,
+                    "format": "jsonlines",
+                    "item_classes": [self.MyItem, self.MyItem2],
+                    "item_filter": CustomFilter3,
                 },
             },
         }
@@ -1129,15 +1145,12 @@ class FeedExportTest(FeedExportTestBase):
         # When dicts are used, only keys from the first row are used as
         # a header for CSV, and all fields are used for JSON Lines.
         items = [
-            {'foo': 'bar', 'egg': 'spam'},
-            {'foo': 'bar', 'egg': 'spam', 'baz': 'quux'},
-        ]
-        rows_csv = [
-            {'egg': 'spam', 'foo': 'bar'},
-            {'egg': 'spam', 'foo': 'bar'}
+            {"foo": "bar", "egg": "spam"},
+            {"foo": "bar", "egg": "spam", "baz": "quux"},
         ]
+        rows_csv = [{"egg": "spam", "foo": "bar"}, {"egg": "spam", "foo": "bar"}]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
+        yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -1147,105 +1160,102 @@ class FeedExportTest(FeedExportTestBase):
 
         for item_cls in [self.MyItem, dict]:
             items = [
-                item_cls({'foo': 'bar1', 'egg': 'spam1'}),
-                item_cls({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+                item_cls({"foo": "bar1", "egg": "spam1"}),
+                item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
             ]
 
             # export all columns
-            settings = {'FEED_EXPORT_FIELDS': 'foo,baz,egg'}
+            settings = {"FEED_EXPORT_FIELDS": "foo,baz,egg"}
             rows = [
-                {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
-                {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
+                {"egg": "spam1", "foo": "bar1", "baz": ""},
+                {"egg": "spam2", "foo": "bar2", "baz": "quux2"},
             ]
-            yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
-                                      settings=settings)
+            yield self.assertExported(
+                items, ["foo", "baz", "egg"], rows, settings=settings
+            )
 
             # export a subset of columns
-            settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
-            rows = [
-                {'egg': 'spam1', 'baz': ''},
-                {'egg': 'spam2', 'baz': 'quux2'}
-            ]
-            yield self.assertExported(items, ['egg', 'baz'], rows,
-                                      settings=settings)
+            settings = {"FEED_EXPORT_FIELDS": "egg,baz"}
+            rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}]
+            yield self.assertExported(items, ["egg", "baz"], rows, settings=settings)
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
-        items = [dict({'foo': 'Test\xd6'})]
+        items = [dict({"foo": "Test\xd6"})]
 
         formats = {
-            'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
-            'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': (
+            "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"),
+            "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"),
+            "xml": (
                 '<?xml version="1.0" encoding="utf-8"?>\n'
-                '<items><item><foo>Test\xd6</foo></item></items>'
-            ).encode('utf-8'),
-            'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'),
+                "<items><item><foo>Test\xd6</foo></item></items>"
+            ).encode("utf-8"),
+            "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"),
         }
 
         for fmt, expected in formats.items():
             settings = {
-                'FEEDS': {
-                    self._random_temp_filename(): {'format': fmt},
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": fmt},
                 },
-                'FEED_EXPORT_INDENT': None,
+                "FEED_EXPORT_INDENT": None,
             }
             data = yield self.exported_data(items, settings)
             self.assertEqual(expected, data[fmt])
 
         formats = {
-            'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'),
-            'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': (
+            "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"),
+            "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"),
+            "xml": (
                 '<?xml version="1.0" encoding="latin-1"?>\n'
-                '<items><item><foo>Test\xd6</foo></item></items>'
-            ).encode('latin-1'),
-            'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'),
+                "<items><item><foo>Test\xd6</foo></item></items>"
+            ).encode("latin-1"),
+            "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"),
         }
 
         for fmt, expected in formats.items():
             settings = {
-                'FEEDS': {
-                    self._random_temp_filename(): {'format': fmt},
+                "FEEDS": {
+                    self._random_temp_filename(): {"format": fmt},
                 },
-                'FEED_EXPORT_INDENT': None,
-                'FEED_EXPORT_ENCODING': 'latin-1',
+                "FEED_EXPORT_INDENT": None,
+                "FEED_EXPORT_ENCODING": "latin-1",
             }
             data = yield self.exported_data(items, settings)
             self.assertEqual(expected, data[fmt])
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
+        items = [dict({"foo": "FOO", "bar": "BAR"})]
 
         formats = {
-            'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': (
+            "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
+            "xml": (
                 '<?xml version="1.0" encoding="latin-1"?>\n'
-                '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
-            ).encode('latin-1'),
-            'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                "<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
+            ).encode("latin-1"),
+            "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"),
         }
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 self._random_temp_filename(): {
-                    'format': 'json',
-                    'indent': 0,
-                    'fields': ['bar'],
-                    'encoding': 'utf-8',
+                    "format": "json",
+                    "indent": 0,
+                    "fields": ["bar"],
+                    "encoding": "utf-8",
                 },
                 self._random_temp_filename(): {
-                    'format': 'xml',
-                    'indent': 2,
-                    'fields': ['foo'],
-                    'encoding': 'latin-1',
+                    "format": "xml",
+                    "indent": 2,
+                    "fields": ["foo"],
+                    "encoding": "latin-1",
                 },
                 self._random_temp_filename(): {
-                    'format': 'csv',
-                    'indent': None,
-                    'fields': ['bar', 'foo'],
-                    'encoding': 'utf-8',
+                    "format": "csv",
+                    "indent": None,
+                    "fields": ["bar", "foo"],
+                    "encoding": "utf-8",
                 },
             },
         }
@@ -1257,37 +1267,37 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_indentation(self):
         items = [
-            {'foo': ['bar']},
-            {'key': 'value'},
+            {"foo": ["bar"]},
+            {"key": "value"},
         ]
 
         test_cases = [
             # JSON
             {
-                'format': 'json',
-                'indent': None,
-                'expected': b'[{"foo": ["bar"]},{"key": "value"}]',
+                "format": "json",
+                "indent": None,
+                "expected": b'[{"foo": ["bar"]},{"key": "value"}]',
             },
             {
-                'format': 'json',
-                'indent': -1,
-                'expected': b"""[
+                "format": "json",
+                "indent": -1,
+                "expected": b"""[
 {"foo": ["bar"]},
 {"key": "value"}
 ]""",
             },
             {
-                'format': 'json',
-                'indent': 0,
-                'expected': b"""[
+                "format": "json",
+                "indent": 0,
+                "expected": b"""[
 {"foo": ["bar"]},
 {"key": "value"}
 ]""",
             },
             {
-                'format': 'json',
-                'indent': 2,
-                'expected': b"""[
+                "format": "json",
+                "indent": 2,
+                "expected": b"""[
 {
   "foo": [
     "bar"
@@ -1299,9 +1309,9 @@ class FeedExportTest(FeedExportTestBase):
 ]""",
             },
             {
-                'format': 'json',
-                'indent': 4,
-                'expected': b"""[
+                "format": "json",
+                "indent": 4,
+                "expected": b"""[
 {
     "foo": [
         "bar"
@@ -1313,9 +1323,9 @@ class FeedExportTest(FeedExportTestBase):
 ]""",
             },
             {
-                'format': 'json',
-                'indent': 5,
-                'expected': b"""[
+                "format": "json",
+                "indent": 5,
+                "expected": b"""[
 {
      "foo": [
           "bar"
@@ -1326,36 +1336,35 @@ class FeedExportTest(FeedExportTestBase):
 }
 ]""",
             },
-
             # XML
             {
-                'format': 'xml',
-                'indent': None,
-                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
+                "format": "xml",
+                "indent": None,
+                "expected": b"""<?xml version="1.0" encoding="utf-8"?>
 <items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",
             },
             {
-                'format': 'xml',
-                'indent': -1,
-                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
+                "format": "xml",
+                "indent": -1,
+                "expected": b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
 <item><foo><value>bar</value></foo></item>
 <item><key>value</key></item>
 </items>""",
             },
             {
-                'format': 'xml',
-                'indent': 0,
-                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
+                "format": "xml",
+                "indent": 0,
+                "expected": b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
 <item><foo><value>bar</value></foo></item>
 <item><key>value</key></item>
 </items>""",
             },
             {
-                'format': 'xml',
-                'indent': 2,
-                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
+                "format": "xml",
+                "indent": 2,
+                "expected": b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
   <item>
     <foo>
@@ -1368,9 +1377,9 @@ class FeedExportTest(FeedExportTestBase):
 </items>""",
             },
             {
-                'format': 'xml',
-                'indent': 4,
-                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
+                "format": "xml",
+                "indent": 4,
+                "expected": b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
     <item>
         <foo>
@@ -1383,9 +1392,9 @@ class FeedExportTest(FeedExportTestBase):
 </items>""",
             },
             {
-                'format': 'xml',
-                'indent': 5,
-                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
+                "format": "xml",
+                "indent": 5,
+                "expected": b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
      <item>
           <foo>
@@ -1401,23 +1410,23 @@ class FeedExportTest(FeedExportTestBase):
 
         for row in test_cases:
             settings = {
-                'FEEDS': {
+                "FEEDS": {
                     self._random_temp_filename(): {
-                        'format': row['format'],
-                        'indent': row['indent'],
+                        "format": row["format"],
+                        "indent": row["indent"],
                     },
                 },
             }
             data = yield self.exported_data(items, settings)
-            self.assertEqual(row['expected'], data[row['format']])
+            self.assertEqual(row["expected"], data[row["format"]])
 
     @defer.inlineCallbacks
     def test_init_exporters_storages_with_crawler(self):
         settings = {
-            'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter},
-            'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage},
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'csv'},
+            "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter},
+            "FEED_STORAGES": {"file": FromCrawlerFileFeedStorage},
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "csv"},
             },
         }
         yield self.exported_data(items=[], settings=settings)
@@ -1427,105 +1436,103 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_str_uri(self):
         settings = {
-            'FEED_STORE_EMPTY': True,
-            'FEEDS': {
-                str(self._random_temp_filename()): {'format': 'csv'}
-            },
+            "FEED_STORE_EMPTY": True,
+            "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}},
         }
         data = yield self.exported_no_data(settings)
-        self.assertEqual(data['csv'], b'')
+        self.assertEqual(data["csv"], b"")
 
     @defer.inlineCallbacks
     def test_multiple_feeds_success_logs_blocking_feed_storage(self):
         settings = {
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'json'},
-                self._random_temp_filename(): {'format': 'xml'},
-                self._random_temp_filename(): {'format': 'csv'},
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+                self._random_temp_filename(): {"format": "xml"},
+                self._random_temp_filename(): {"format": "csv"},
             },
-            'FEED_STORAGES': {'file': DummyBlockingFeedStorage},
+            "FEED_STORAGES": {"file": DummyBlockingFeedStorage},
         }
         items = [
-            {'foo': 'bar1', 'baz': ''},
-            {'foo': 'bar2', 'baz': 'quux'},
+            {"foo": "bar1", "baz": ""},
+            {"foo": "bar2", "baz": "quux"},
         ]
         with LogCapture() as log:
             yield self.exported_data(items, settings)
 
         print(log)
-        for fmt in ['json', 'xml', 'csv']:
-            self.assertIn(f'Stored {fmt} feed (2 items)', str(log))
+        for fmt in ["json", "xml", "csv"]:
+            self.assertIn(f"Stored {fmt} feed (2 items)", str(log))
 
     @defer.inlineCallbacks
     def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
         settings = {
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'json'},
-                self._random_temp_filename(): {'format': 'xml'},
-                self._random_temp_filename(): {'format': 'csv'},
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+                self._random_temp_filename(): {"format": "xml"},
+                self._random_temp_filename(): {"format": "csv"},
             },
-            'FEED_STORAGES': {'file': FailingBlockingFeedStorage},
+            "FEED_STORAGES": {"file": FailingBlockingFeedStorage},
         }
         items = [
-            {'foo': 'bar1', 'baz': ''},
-            {'foo': 'bar2', 'baz': 'quux'},
+            {"foo": "bar1", "baz": ""},
+            {"foo": "bar2", "baz": "quux"},
         ]
         with LogCapture() as log:
             yield self.exported_data(items, settings)
 
         print(log)
-        for fmt in ['json', 'xml', 'csv']:
-            self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
+        for fmt in ["json", "xml", "csv"]:
+            self.assertIn(f"Error storing {fmt} feed (2 items)", str(log))
 
     @defer.inlineCallbacks
     def test_extend_kwargs(self):
-        items = [{'foo': 'FOO', 'bar': 'BAR'}]
+        items = [{"foo": "FOO", "bar": "BAR"}]
 
-        expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
-        expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
+        expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8")
+        expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8")
         test_cases = [
             # with title
             {
-                'options': {
-                    'format': 'csv',
-                    'item_export_kwargs': {'include_headers_line': True},
+                "options": {
+                    "format": "csv",
+                    "item_export_kwargs": {"include_headers_line": True},
                 },
-                'expected': expected_with_title_csv,
+                "expected": expected_with_title_csv,
             },
             # without title
             {
-                'options': {
-                    'format': 'csv',
-                    'item_export_kwargs': {'include_headers_line': False},
+                "options": {
+                    "format": "csv",
+                    "item_export_kwargs": {"include_headers_line": False},
                 },
-                'expected': expected_without_title_csv,
+                "expected": expected_without_title_csv,
             },
         ]
 
         for row in test_cases:
-            feed_options = row['options']
+            feed_options = row["options"]
             settings = {
-                'FEEDS': {
+                "FEEDS": {
                     self._random_temp_filename(): feed_options,
                 },
-                'FEED_EXPORT_INDENT': None,
+                "FEED_EXPORT_INDENT": None,
             }
 
             data = yield self.exported_data(items, settings)
-            self.assertEqual(row['expected'], data[feed_options['format']])
+            self.assertEqual(row["expected"], data[feed_options["format"]])
 
 
 class FeedPostProcessedExportsTest(FeedExportTestBase):
     __test__ = True
 
-    items = [{'foo': 'bar'}]
-    expected = b'foo\r\nbar\r\n'
+    items = [{"foo": "bar"}]
+    expected = b"foo\r\nbar\r\n"
 
     class MyPlugin1:
         def __init__(self, file, feed_options):
             self.file = file
             self.feed_options = feed_options
-            self.char = self.feed_options.get('plugin1_char', b'')
+            self.char = self.feed_options.get("plugin1_char", b"")
 
         def write(self, data):
             written_count = self.file.write(data)
@@ -1540,10 +1547,10 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
-        """ Run spider with specified settings; return exported data with filename. """
+        """Run spider with specified settings; return exported data with filename."""
 
-        FEEDS = settings.get('FEEDS') or {}
-        settings['FEEDS'] = {
+        FEEDS = settings.get("FEEDS") or {}
+        settings["FEEDS"] = {
             printf_escape(path_to_url(file_path)): feed_options
             for file_path, feed_options in FEEDS.items()
         }
@@ -1551,7 +1558,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
         content = {}
         try:
             with MockServer() as s:
-                spider_cls.start_urls = [s.url('/')]
+                spider_cls.start_urls = [s.url("/")]
                 crawler = get_crawler(spider_cls, settings)
                 yield crawler.crawl()
 
@@ -1570,10 +1577,15 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
         return content
 
-    def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''):
+    def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""):
         data_stream = BytesIO()
-        gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime,
-                              compresslevel=compresslevel, mode="wb")
+        gzipf = gzip.GzipFile(
+            fileobj=data_stream,
+            filename=filename,
+            mtime=mtime,
+            compresslevel=compresslevel,
+            mode="wb",
+        )
         gzipf.write(data)
         gzipf.close()
         data_stream.seek(0)
@@ -1582,13 +1594,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_gzip_plugin(self):
 
-        filename = self._named_tempfile('gzip_file')
+        filename = self._named_tempfile("gzip_file")
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 filename: {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
                 },
             },
         }
@@ -1603,25 +1615,29 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     def test_gzip_plugin_compresslevel(self):
 
         filename_to_compressed = {
-            self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0),
-            self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9),
+            self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
+                self.expected, compresslevel=0
+            ),
+            self._named_tempfile("compresslevel_9"): self.get_gzip_compressed(
+                self.expected, compresslevel=9
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('compresslevel_0'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
-                    'gzip_compresslevel': 0,
-                    'gzip_mtime': 0,
-                    'gzip_filename': "",
+            "FEEDS": {
+                self._named_tempfile("compresslevel_0"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
+                    "gzip_compresslevel": 0,
+                    "gzip_mtime": 0,
+                    "gzip_filename": "",
                 },
-                self._named_tempfile('compresslevel_9'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
-                    'gzip_compresslevel': 9,
-                    'gzip_mtime': 0,
-                    'gzip_filename': "",
+                self._named_tempfile("compresslevel_9"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
+                    "gzip_compresslevel": 9,
+                    "gzip_mtime": 0,
+                    "gzip_filename": "",
                 },
             },
         }
@@ -1636,23 +1652,27 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_gzip_plugin_mtime(self):
         filename_to_compressed = {
-            self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123),
-            self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789),
+            self._named_tempfile("mtime_123"): self.get_gzip_compressed(
+                self.expected, mtime=123
+            ),
+            self._named_tempfile("mtime_123456789"): self.get_gzip_compressed(
+                self.expected, mtime=123456789
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('mtime_123'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
-                    'gzip_mtime': 123,
-                    'gzip_filename': "",
+            "FEEDS": {
+                self._named_tempfile("mtime_123"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
+                    "gzip_mtime": 123,
+                    "gzip_filename": "",
                 },
-                self._named_tempfile('mtime_123456789'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
-                    'gzip_mtime': 123456789,
-                    'gzip_filename': "",
+                self._named_tempfile("mtime_123456789"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
+                    "gzip_mtime": 123456789,
+                    "gzip_filename": "",
                 },
             },
         }
@@ -1667,23 +1687,27 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_gzip_plugin_filename(self):
         filename_to_compressed = {
-            self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"),
-            self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"),
+            self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(
+                self.expected, filename="FILE1"
+            ),
+            self._named_tempfile("filename_FILE2"): self.get_gzip_compressed(
+                self.expected, filename="FILE2"
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('filename_FILE1'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
-                    'gzip_mtime': 0,
-                    'gzip_filename': "FILE1",
+            "FEEDS": {
+                self._named_tempfile("filename_FILE1"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
+                    "gzip_mtime": 0,
+                    "gzip_filename": "FILE1",
                 },
-                self._named_tempfile('filename_FILE2'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
-                    'gzip_mtime': 0,
-                    'gzip_filename': "FILE2",
+                self._named_tempfile("filename_FILE2"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
+                    "gzip_mtime": 0,
+                    "gzip_filename": "FILE2",
                 },
             },
         }
@@ -1698,13 +1722,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_lzma_plugin(self):
 
-        filename = self._named_tempfile('lzma_file')
+        filename = self._named_tempfile("lzma_file")
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 filename: {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
                 },
             },
         }
@@ -1719,21 +1743,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     def test_lzma_plugin_format(self):
 
         filename_to_compressed = {
-            self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ),
-            self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE),
+            self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
+                self.expected, format=lzma.FORMAT_XZ
+            ),
+            self._named_tempfile("format_FORMAT_ALONE"): lzma.compress(
+                self.expected, format=lzma.FORMAT_ALONE
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('format_FORMAT_XZ'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_format': lzma.FORMAT_XZ,
+            "FEEDS": {
+                self._named_tempfile("format_FORMAT_XZ"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_format": lzma.FORMAT_XZ,
                 },
-                self._named_tempfile('format_FORMAT_ALONE'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_format': lzma.FORMAT_ALONE,
+                self._named_tempfile("format_FORMAT_ALONE"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_format": lzma.FORMAT_ALONE,
                 },
             },
         }
@@ -1749,21 +1777,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     def test_lzma_plugin_check(self):
 
         filename_to_compressed = {
-            self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE),
-            self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256),
+            self._named_tempfile("check_CHECK_NONE"): lzma.compress(
+                self.expected, check=lzma.CHECK_NONE
+            ),
+            self._named_tempfile("check_CHECK_CRC256"): lzma.compress(
+                self.expected, check=lzma.CHECK_SHA256
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('check_CHECK_NONE'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_check': lzma.CHECK_NONE,
+            "FEEDS": {
+                self._named_tempfile("check_CHECK_NONE"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_check": lzma.CHECK_NONE,
                 },
-                self._named_tempfile('check_CHECK_CRC256'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_check': lzma.CHECK_SHA256,
+                self._named_tempfile("check_CHECK_CRC256"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_check": lzma.CHECK_SHA256,
                 },
             },
         }
@@ -1779,21 +1811,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     def test_lzma_plugin_preset(self):
 
         filename_to_compressed = {
-            self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0),
-            self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9),
+            self._named_tempfile("preset_PRESET_0"): lzma.compress(
+                self.expected, preset=0
+            ),
+            self._named_tempfile("preset_PRESET_9"): lzma.compress(
+                self.expected, preset=9
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('preset_PRESET_0'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_preset': 0,
+            "FEEDS": {
+                self._named_tempfile("preset_PRESET_0"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_preset": 0,
                 },
-                self._named_tempfile('preset_PRESET_9'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_preset': 9,
+                self._named_tempfile("preset_PRESET_9"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_preset": 9,
                 },
             },
         }
@@ -1811,16 +1847,16 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
             # https://foss.heptapod.net/pypy/pypy/-/issues/3527
             raise unittest.SkipTest("lzma filters doesn't work in PyPy")
 
-        filters = [{'id': lzma.FILTER_LZMA2}]
+        filters = [{"id": lzma.FILTER_LZMA2}]
         compressed = lzma.compress(self.expected, filters=filters)
-        filename = self._named_tempfile('filters')
+        filename = self._named_tempfile("filters")
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 filename: {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'lzma_filters': filters,
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
+                    "lzma_filters": filters,
                 },
             },
         }
@@ -1833,13 +1869,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_bz2_plugin(self):
 
-        filename = self._named_tempfile('bz2_file')
+        filename = self._named_tempfile("bz2_file")
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 filename: {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
                 },
             },
         }
@@ -1854,21 +1890,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     def test_bz2_plugin_compresslevel(self):
 
         filename_to_compressed = {
-            self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1),
-            self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9),
+            self._named_tempfile("compresslevel_1"): bz2.compress(
+                self.expected, compresslevel=1
+            ),
+            self._named_tempfile("compresslevel_9"): bz2.compress(
+                self.expected, compresslevel=9
+            ),
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('compresslevel_1'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
-                    'bz2_compresslevel': 1,
+            "FEEDS": {
+                self._named_tempfile("compresslevel_1"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
+                    "bz2_compresslevel": 1,
                 },
-                self._named_tempfile('compresslevel_9'): {
-                    'format': 'csv',
-                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
-                    'bz2_compresslevel': 9,
+                self._named_tempfile("compresslevel_9"): {
+                    "format": "csv",
+                    "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
+                    "bz2_compresslevel": 9,
                 },
             },
         }
@@ -1882,13 +1922,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_custom_plugin(self):
-        filename = self._named_tempfile('csv_file')
+        filename = self._named_tempfile("csv_file")
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 filename: {
-                    'format': 'csv',
-                    'postprocessing': [self.MyPlugin1],
+                    "format": "csv",
+                    "postprocessing": [self.MyPlugin1],
                 },
             },
         }
@@ -1899,15 +1939,15 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_custom_plugin_with_parameter(self):
 
-        expected = b'foo\r\n\nbar\r\n\n'
-        filename = self._named_tempfile('newline')
+        expected = b"foo\r\n\nbar\r\n\n"
+        filename = self._named_tempfile("newline")
 
         settings = {
-            'FEEDS': {
+            "FEEDS": {
                 filename: {
-                    'format': 'csv',
-                    'postprocessing': [self.MyPlugin1],
-                    'plugin1_char': b'\n'
+                    "format": "csv",
+                    "postprocessing": [self.MyPlugin1],
+                    "plugin1_char": b"\n",
                 },
             },
         }
@@ -1918,30 +1958,39 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_custom_plugin_with_compression(self):
 
-        expected = b'foo\r\n\nbar\r\n\n'
+        expected = b"foo\r\n\nbar\r\n\n"
 
         filename_to_decompressor = {
-            self._named_tempfile('bz2'): bz2.decompress,
-            self._named_tempfile('lzma'): lzma.decompress,
-            self._named_tempfile('gzip'): gzip.decompress,
+            self._named_tempfile("bz2"): bz2.decompress,
+            self._named_tempfile("lzma"): lzma.decompress,
+            self._named_tempfile("gzip"): gzip.decompress,
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('bz2'): {
-                    'format': 'csv',
-                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'],
-                    'plugin1_char': b'\n',
+            "FEEDS": {
+                self._named_tempfile("bz2"): {
+                    "format": "csv",
+                    "postprocessing": [
+                        self.MyPlugin1,
+                        "scrapy.extensions.postprocessing.Bz2Plugin",
+                    ],
+                    "plugin1_char": b"\n",
                 },
-                self._named_tempfile('lzma'): {
-                    'format': 'csv',
-                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'],
-                    'plugin1_char': b'\n',
+                self._named_tempfile("lzma"): {
+                    "format": "csv",
+                    "postprocessing": [
+                        self.MyPlugin1,
+                        "scrapy.extensions.postprocessing.LZMAPlugin",
+                    ],
+                    "plugin1_char": b"\n",
                 },
-                self._named_tempfile('gzip'): {
-                    'format': 'csv',
-                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
-                    'plugin1_char': b'\n',
+                self._named_tempfile("gzip"): {
+                    "format": "csv",
+                    "postprocessing": [
+                        self.MyPlugin1,
+                        "scrapy.extensions.postprocessing.GzipPlugin",
+                    ],
+                    "plugin1_char": b"\n",
                 },
             },
         }
@@ -1956,40 +2005,41 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
     def test_exports_compatibility_with_postproc(self):
         import marshal
         import pickle
+
         filename_to_expected = {
-            self._named_tempfile('csv'): b'foo\r\nbar\r\n',
-            self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]',
-            self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n',
-            self._named_tempfile('xml'): b'<?xml version="1.0" encoding="utf-8"?>\n'
-                                         b'<items>\n<item><foo>bar</foo></item>\n</items>',
+            self._named_tempfile("csv"): b"foo\r\nbar\r\n",
+            self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',
+            self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n',
+            self._named_tempfile("xml"): b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b"<items>\n<item><foo>bar</foo></item>\n</items>",
         }
 
         settings = {
-            'FEEDS': {
-                self._named_tempfile('csv'): {
-                    'format': 'csv',
-                    'postprocessing': [self.MyPlugin1],
+            "FEEDS": {
+                self._named_tempfile("csv"): {
+                    "format": "csv",
+                    "postprocessing": [self.MyPlugin1],
                     # empty plugin to activate postprocessing.PostProcessingManager
                 },
-                self._named_tempfile('json'): {
-                    'format': 'json',
-                    'postprocessing': [self.MyPlugin1],
+                self._named_tempfile("json"): {
+                    "format": "json",
+                    "postprocessing": [self.MyPlugin1],
                 },
-                self._named_tempfile('jsonlines'): {
-                    'format': 'jsonlines',
-                    'postprocessing': [self.MyPlugin1],
+                self._named_tempfile("jsonlines"): {
+                    "format": "jsonlines",
+                    "postprocessing": [self.MyPlugin1],
                 },
-                self._named_tempfile('xml'): {
-                    'format': 'xml',
-                    'postprocessing': [self.MyPlugin1],
+                self._named_tempfile("xml"): {
+                    "format": "xml",
+                    "postprocessing": [self.MyPlugin1],
                 },
-                self._named_tempfile('marshal'): {
-                    'format': 'marshal',
-                    'postprocessing': [self.MyPlugin1],
+                self._named_tempfile("marshal"): {
+                    "format": "marshal",
+                    "postprocessing": [self.MyPlugin1],
                 },
-                self._named_tempfile('pickle'): {
-                    'format': 'pickle',
-                    'postprocessing': [self.MyPlugin1],
+                self._named_tempfile("pickle"): {
+                    "format": "pickle",
+                    "postprocessing": [self.MyPlugin1],
                 },
             },
         }
@@ -1997,9 +2047,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
         data = yield self.exported_data(self.items, settings)
 
         for filename, result in data.items():
-            if 'pickle' in filename:
+            if "pickle" in filename:
                 expected, result = self.items[0], pickle.loads(result)
-            elif 'marshal' in filename:
+            elif "marshal" in filename:
                 expected, result = self.items[0], marshal.loads(result)
             else:
                 expected = filename_to_expected[filename]
@@ -2008,28 +2058,27 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(batch_time)s_#%(batch_id)02d_'
+    _file_mark = "_%(batch_time)s_#%(batch_id)02d_"
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
-        """ Run spider with specified settings; return exported data. """
+        """Run spider with specified settings; return exported data."""
 
-        FEEDS = settings.get('FEEDS') or {}
-        settings['FEEDS'] = {
-            build_url(file_path): feed
-            for file_path, feed in FEEDS.items()
+        FEEDS = settings.get("FEEDS") or {}
+        settings["FEEDS"] = {
+            build_url(file_path): feed for file_path, feed in FEEDS.items()
         }
         content = defaultdict(list)
         try:
             with MockServer() as s:
-                spider_cls.start_urls = [s.url('/')]
+                spider_cls.start_urls = [s.url("/")]
                 crawler = get_crawler(spider_cls, settings)
                 yield crawler.crawl()
 
             for path, feed in FEEDS.items():
                 dir_name = Path(path).parent
                 for file in sorted(dir_name.iterdir()):
-                    content[feed['format']].append(file.read_bytes())
+                    content[feed["format"]].append(file.read_bytes())
         finally:
             self.tearDown()
         defer.returnValue(content)
@@ -2037,30 +2086,40 @@ class BatchDeliveriesTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'},
-            },
-        })
-        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / "jl"
+                    / self._file_mark: {"format": "jl"},
+                },
+            }
+        )
+        batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
-        for batch in data['jl']:
-            got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]
+        for batch in data["jl"]:
+            got_batch = [
+                json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()
+            ]
             expected_batch, rows = rows[:batch_size], rows[batch_size:]
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'},
-            },
-        })
-        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / "csv"
+                    / self._file_mark: {"format": "csv"},
+                },
+            }
+        )
+        batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         data = yield self.exported_data(items, settings)
-        for batch in data['csv']:
+        for batch in data["csv"]:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
             self.assertEqual(list(header), got_batch.fieldnames)
             expected_batch, rows = rows[:batch_size], rows[batch_size:]
@@ -2069,59 +2128,74 @@ class BatchDeliveriesTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def assertExportedXml(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
-            },
-        })
-        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / "xml"
+                    / self._file_mark: {"format": "xml"},
+                },
+            }
+        )
+        batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
-        for batch in data['xml']:
+        for batch in data["xml"]:
             root = lxml.etree.fromstring(batch)
-            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")]
             expected_batch, rows = rows[:batch_size], rows[batch_size:]
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
     def assertExportedMultiple(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
-                self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'},
-            },
-        })
-        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / "xml"
+                    / self._file_mark: {"format": "xml"},
+                    self._random_temp_filename()
+                    / "json"
+                    / self._file_mark: {"format": "json"},
+                },
+            }
+        )
+        batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
         xml_rows = rows.copy()
-        for batch in data['xml']:
+        for batch in data["xml"]:
             root = lxml.etree.fromstring(batch)
-            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")]
             expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
             self.assertEqual(expected_batch, got_batch)
         # JSON
         json_rows = rows.copy()
-        for batch in data['json']:
-            got_batch = json.loads(batch.decode('utf-8'))
+        for batch in data["json"]:
+            got_batch = json.loads(batch.decode("utf-8"))
             expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
     def assertExportedPickle(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'},
-            },
-        })
-        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / "pickle"
+                    / self._file_mark: {"format": "pickle"},
+                },
+            }
+        )
+        batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
-        for batch in data['pickle']:
+
+        for batch in data["pickle"]:
             got_batch = self._load_until_eof(batch, load_func=pickle.load)
             expected_batch, rows = rows[:batch_size], rows[batch_size:]
             self.assertEqual(expected_batch, got_batch)
@@ -2129,80 +2203,87 @@ class BatchDeliveriesTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def assertExportedMarshal(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({
-            'FEEDS': {
-                self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'},
-            },
-        })
-        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        settings.update(
+            {
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / "marshal"
+                    / self._file_mark: {"format": "marshal"},
+                },
+            }
+        )
+        batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
-        for batch in data['marshal']:
+
+        for batch in data["marshal"]:
             got_batch = self._load_until_eof(batch, load_func=marshal.load)
             expected_batch, rows = rows[:batch_size], rows[batch_size:]
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
     def test_export_items(self):
-        """ Test partial deliveries in all supported formats """
+        """Test partial deliveries in all supported formats"""
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
-            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            self.MyItem({"foo": "bar3", "baz": "quux3"}),
         ]
         rows = [
-            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
-            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
+            {"egg": "spam1", "foo": "bar1", "baz": ""},
+            {"egg": "spam2", "foo": "bar2", "baz": "quux2"},
+            {"foo": "bar3", "baz": "quux3", "egg": ""},
         ]
-        settings = {
-            'FEED_EXPORT_BATCH_ITEM_COUNT': 2
-        }
+        settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """
+        """If path is without %(batch_time)s and %(batch_id) an exception must be raised"""
         settings = {
-            'FEEDS': {
-                self._random_temp_filename(): {'format': 'xml'},
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "xml"},
             },
-            'FEED_EXPORT_BATCH_ITEM_COUNT': 1
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
-        for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+        for fmt in ("json", "jsonlines", "xml", "csv"):
             settings = {
-                'FEEDS': {
-                    self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / fmt
+                    / self._file_mark: {"format": fmt},
                 },
-                'FEED_EXPORT_BATCH_ITEM_COUNT': 1
+                "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
-            self.assertEqual(b'', data[fmt][0])
+            self.assertEqual(b"", data[fmt][0])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
         formats = (
-            ('json', b'[]'),
-            ('jsonlines', b''),
-            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
-            ('csv', b''),
+            ("json", b"[]"),
+            ("jsonlines", b""),
+            ("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ("csv", b""),
         )
 
         for fmt, expctd in formats:
             settings = {
-                'FEEDS': {
-                    self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
+                "FEEDS": {
+                    self._random_temp_filename()
+                    / fmt
+                    / self._file_mark: {"format": fmt},
                 },
-                'FEED_STORE_EMPTY': True,
-                'FEED_EXPORT_INDENT': None,
-                'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
+                "FEED_STORE_EMPTY": True,
+                "FEED_EXPORT_INDENT": None,
+                "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -2210,47 +2291,60 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
+        items = [
+            dict({"foo": "FOO", "bar": "BAR"}),
+            dict({"foo": "FOO1", "bar": "BAR1"}),
+        ]
 
         formats = {
-            'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-                     '[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
-            'xml': [
-                (
-                    '<?xml version="1.0" encoding="latin-1"?>\n'
-                    '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
-                ).encode('latin-1'),
-                (
-                    '<?xml version="1.0" encoding="latin-1"?>\n'
-                    '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
-                ).encode('latin-1')
+            "json": [
+                '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
+                '[\n{"bar": "BAR1"}\n]'.encode("utf-8"),
+            ],
+            "xml": [
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    "<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
+                ).encode("latin-1"),
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    "<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>"
+                ).encode("latin-1"),
+            ],
+            "csv": [
+                "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"),
+                "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"),
             ],
-            'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
-                    'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
         }
 
         settings = {
-            'FEEDS': {
-                self._random_temp_filename() / 'json' / self._file_mark: {
-                    'format': 'json',
-                    'indent': 0,
-                    'fields': ['bar'],
-                    'encoding': 'utf-8',
+            "FEEDS": {
+                self._random_temp_filename()
+                / "json"
+                / self._file_mark: {
+                    "format": "json",
+                    "indent": 0,
+                    "fields": ["bar"],
+                    "encoding": "utf-8",
                 },
-                self._random_temp_filename() / 'xml' / self._file_mark: {
-                    'format': 'xml',
-                    'indent': 2,
-                    'fields': ['foo'],
-                    'encoding': 'latin-1',
+                self._random_temp_filename()
+                / "xml"
+                / self._file_mark: {
+                    "format": "xml",
+                    "indent": 2,
+                    "fields": ["foo"],
+                    "encoding": "latin-1",
                 },
-                self._random_temp_filename() / 'csv' / self._file_mark: {
-                    'format': 'csv',
-                    'indent': None,
-                    'fields': ['foo', 'bar'],
-                    'encoding': 'utf-8',
+                self._random_temp_filename()
+                / "csv"
+                / self._file_mark: {
+                    "format": "csv",
+                    "indent": None,
+                    "fields": ["foo", "bar"],
+                    "encoding": "utf-8",
                 },
             },
-            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -2259,18 +2353,22 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
+        items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})]
         formats = {
-            'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
-                     '[{"foo": "FOO1"}]'.encode('utf-8')],
+            "json": [
+                '[{"foo": "FOO"}]'.encode("utf-8"),
+                '[{"foo": "FOO1"}]'.encode("utf-8"),
+            ],
         }
         settings = {
-            'FEEDS': {
-                self._random_temp_filename() / 'json' / self._file_mark: {
-                    'format': 'json',
-                    'indent': None,
-                    'encoding': 'utf-8',
-                    'batch_item_count': 1,
+            "FEEDS": {
+                self._random_temp_filename()
+                / "json"
+                / self._file_mark: {
+                    "format": "json",
+                    "indent": None,
+                    "encoding": "utf-8",
+                    "batch_item_count": 1,
                 },
             },
         }
@@ -2279,7 +2377,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
             for expected_batch, got_batch in zip(expected, data[fmt]):
                 self.assertEqual(expected_batch, got_batch)
 
-    @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output')
+    @pytest.mark.skipif(
+        sys.platform == "win32", reason="Odd behaviour on file creation/output"
+    )
     @defer.inlineCallbacks
     def test_batch_path_differ(self):
         """
@@ -2287,26 +2387,29 @@ class BatchDeliveriesTest(FeedExportTestBase):
         So %(batch_time)s replaced with the current date.
         """
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
-            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            self.MyItem({"foo": "bar3", "baz": "quux3"}),
         ]
         settings = {
-            'FEEDS': {
-                self._random_temp_filename() / '%(batch_time)s': {
-                    'format': 'json',
+            "FEEDS": {
+                self._random_temp_filename()
+                / "%(batch_time)s": {
+                    "format": "json",
                 },
             },
-            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
         }
         data = yield self.exported_data(items, settings)
-        self.assertEqual(len(items), len([_ for _ in data['json'] if _]))
+        self.assertEqual(len(items), len([_ for _ in data["json"] if _]))
 
     @defer.inlineCallbacks
     def test_stats_batch_file_success(self):
         settings = {
             "FEEDS": {
-                build_url(str(self._random_temp_filename() / "json" / self._file_mark)): {
+                build_url(
+                    str(self._random_temp_filename() / "json" / self._file_mark)
+                ): {
                     "format": "json",
                 }
             },
@@ -2315,18 +2418,22 @@ class BatchDeliveriesTest(FeedExportTestBase):
         crawler = get_crawler(ItemSpider, settings)
         with MockServer() as mockserver:
             yield crawler.crawl(total=2, mockserver=mockserver)
-        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
-        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12)
+        self.assertIn(
+            "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()
+        )
+        self.assertEqual(
+            crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12
+        )
 
     @defer.inlineCallbacks
     def test_s3_export(self):
         skip_if_no_boto()
 
-        bucket = 'mybucket'
+        bucket = "mybucket"
         items = [
-            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
-            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            self.MyItem({"foo": "bar3", "baz": "quux3"}),
         ]
 
         class CustomS3FeedStorage(S3FeedStorage):
@@ -2335,33 +2442,34 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             def open(self, *args, **kwargs):
                 from botocore.stub import ANY, Stubber
+
                 stub = Stubber(self.s3_client)
                 stub.activate()
                 CustomS3FeedStorage.stubs.append(stub)
                 stub.add_response(
-                    'put_object',
+                    "put_object",
                     expected_params={
-                        'Body': ANY,
-                        'Bucket': bucket,
-                        'Key': ANY,
+                        "Body": ANY,
+                        "Bucket": bucket,
+                        "Key": ANY,
                     },
                     service_response={},
                 )
                 return super().open(*args, **kwargs)
 
-        key = 'export.csv'
-        uri = f's3://{bucket}/{key}/%(batch_time)s.json'
+        key = "export.csv"
+        uri = f"s3://{bucket}/{key}/%(batch_time)s.json"
         batch_item_count = 1
         settings = {
-            'AWS_ACCESS_KEY_ID': 'access_key',
-            'AWS_SECRET_ACCESS_KEY': 'secret_key',
-            'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count,
-            'FEED_STORAGES': {
-                's3': CustomS3FeedStorage,
+            "AWS_ACCESS_KEY_ID": "access_key",
+            "AWS_SECRET_ACCESS_KEY": "secret_key",
+            "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count,
+            "FEED_STORAGES": {
+                "s3": CustomS3FeedStorage,
             },
-            'FEEDS': {
+            "FEEDS": {
                 uri: {
-                    'format': 'json',
+                    "format": "json",
                 },
             },
         }
@@ -2370,14 +2478,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
         verifyObject(IFeedStorage, storage)
 
         class TestSpider(scrapy.Spider):
-            name = 'testspider'
+            name = "testspider"
 
             def parse(self, response):
                 for item in items:
                     yield item
 
         with MockServer() as server:
-            TestSpider.start_urls = [server.url('/')]
+            TestSpider.start_urls = [server.url("/")]
             crawler = get_crawler(TestSpider, settings)
             yield crawler.crawl()
 
@@ -2387,11 +2495,10 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
 
 class FeedExportInitTest(unittest.TestCase):
-
     def test_unsupported_storage(self):
         settings = {
-            'FEEDS': {
-                'unsupported://uri': {},
+            "FEEDS": {
+                "unsupported://uri": {},
             },
         }
         crawler = get_crawler(settings_dict=settings)
@@ -2400,9 +2507,9 @@ class FeedExportInitTest(unittest.TestCase):
 
     def test_unsupported_format(self):
         settings = {
-            'FEEDS': {
-                'file://path': {
-                    'format': 'unsupported_format',
+            "FEEDS": {
+                "file://path": {
+                    "format": "unsupported_format",
                 },
             },
         }
@@ -2412,7 +2519,6 @@ class FeedExportInitTest(unittest.TestCase):
 
 
 class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
-
     def __init__(self, uri):
         super().__init__(uri)
 
@@ -2424,25 +2530,26 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_init(self):
         settings_dict = {
-            'FEED_URI': 'file:///tmp/foobar',
-            'FEED_STORAGES': {
-                'file': StdoutFeedStorageWithoutFeedOptions
-            },
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions},
         }
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
             crawler = get_crawler(settings_dict=settings_dict)
             feed_exporter = FeedExporter.from_crawler(crawler)
 
         spider = scrapy.Spider("default")
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="StdoutFeedStorageWithoutFeedOptions does not support "
-                                "the 'feed_options' keyword argument."):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="StdoutFeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
             feed_exporter.open_spider(spider)
 
 
 class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
-
     def __init__(self, uri):
         super().__init__(uri)
 
@@ -2457,31 +2564,31 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
     def test_init(self):
         with tempfile.NamedTemporaryFile() as temp:
             settings_dict = {
-                'FEED_URI': f'file:///{temp.name}',
-                'FEED_STORAGES': {
-                    'file': FileFeedStorageWithoutFeedOptions
-                },
+                "FEED_URI": f"file:///{temp.name}",
+                "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions},
             }
-            with pytest.warns(ScrapyDeprecationWarning,
-                              match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            with pytest.warns(
+                ScrapyDeprecationWarning,
+                match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+            ):
                 crawler = get_crawler(settings_dict=settings_dict)
                 feed_exporter = FeedExporter.from_crawler(crawler)
         spider = scrapy.Spider("default")
 
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="FileFeedStorageWithoutFeedOptions does not support "
-                                "the 'feed_options' keyword argument."):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="FileFeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
             feed_exporter.open_spider(spider)
 
 
 class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
-
     def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):
         super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)
 
 
 class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
-
     @classmethod
     def from_crawler(cls, crawler, uri):
         return super().from_crawler(crawler, uri)
@@ -2496,53 +2603,55 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_init(self):
         settings_dict = {
-            'FEED_URI': 'file:///tmp/foobar',
-            'FEED_STORAGES': {
-                'file': S3FeedStorageWithoutFeedOptions
-            },
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions},
         }
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
             crawler = get_crawler(settings_dict=settings_dict)
             feed_exporter = FeedExporter.from_crawler(crawler)
 
         spider = scrapy.Spider("default")
         spider.crawler = crawler
 
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="S3FeedStorageWithoutFeedOptions does not support "
-                                "the 'feed_options' keyword argument."):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="S3FeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
             feed_exporter.open_spider(spider)
 
     def test_from_crawler(self):
         settings_dict = {
-            'FEED_URI': 'file:///tmp/foobar',
-            'FEED_STORAGES': {
-                'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
-            },
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler},
         }
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
             crawler = get_crawler(settings_dict=settings_dict)
             feed_exporter = FeedExporter.from_crawler(crawler)
 
         spider = scrapy.Spider("default")
         spider.crawler = crawler
 
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
-                                "the 'feed_options' keyword argument."):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+            "the 'feed_options' keyword argument.",
+        ):
             feed_exporter.open_spider(spider)
 
 
 class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
-
     def __init__(self, uri, use_active_mode=False):
         super().__init__(uri)
 
 
 class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
-
     @classmethod
     def from_crawler(cls, crawler, uri):
         return super().from_crawler(crawler, uri)
@@ -2557,42 +2666,46 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_init(self):
         settings_dict = {
-            'FEED_URI': 'file:///tmp/foobar',
-            'FEED_STORAGES': {
-                'file': FTPFeedStorageWithoutFeedOptions
-            },
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions},
         }
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
             crawler = get_crawler(settings_dict=settings_dict)
             feed_exporter = FeedExporter.from_crawler(crawler)
 
         spider = scrapy.Spider("default")
         spider.crawler = crawler
 
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="FTPFeedStorageWithoutFeedOptions does not support "
-                                "the 'feed_options' keyword argument."):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="FTPFeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
             feed_exporter.open_spider(spider)
 
     def test_from_crawler(self):
         settings_dict = {
-            'FEED_URI': 'file:///tmp/foobar',
-            'FEED_STORAGES': {
-                'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
-            },
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
         }
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
             crawler = get_crawler(settings_dict=settings_dict)
             feed_exporter = FeedExporter.from_crawler(crawler)
 
         spider = scrapy.Spider("default")
         spider.crawler = crawler
 
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
-                                "the 'feed_options' keyword argument."):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+            "the 'feed_options' keyword argument.",
+        ):
             feed_exporter.open_spider(spider)
 
 
@@ -2601,13 +2714,15 @@ class URIParamsTest:
     spider_name = "uri_params_spider"
     deprecated_options = False
 
-    def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
+    def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
         raise NotImplementedError
 
     def _crawler_feed_exporter(self, settings):
         if self.deprecated_options:
-            with pytest.warns(ScrapyDeprecationWarning,
-                              match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            with pytest.warns(
+                ScrapyDeprecationWarning,
+                match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+            ):
                 crawler = get_crawler(settings_dict=settings)
                 feed_exporter = FeedExporter.from_crawler(crawler)
         else:
@@ -2617,7 +2732,7 @@ class URIParamsTest:
 
     def test_default(self):
         settings = self.build_settings(
-            uri='file:///tmp/%(name)s',
+            uri="file:///tmp/%(name)s",
         )
         crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
@@ -2627,38 +2742,33 @@ class URIParamsTest:
             warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
 
-        self.assertEqual(
-            feed_exporter.slots[0].uri,
-            f'file:///tmp/{self.spider_name}'
-        )
+        self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}")
 
     def test_none(self):
         def uri_params(params, spider):
             pass
 
         settings = self.build_settings(
-            uri='file:///tmp/%(name)s',
+            uri="file:///tmp/%(name)s",
             uri_params=uri_params,
         )
         crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
 
-        with pytest.warns(ScrapyDeprecationWarning,
-                          match="Modifying the params dictionary in-place"):
+        with pytest.warns(
+            ScrapyDeprecationWarning, match="Modifying the params dictionary in-place"
+        ):
             feed_exporter.open_spider(spider)
 
-        self.assertEqual(
-            feed_exporter.slots[0].uri,
-            f'file:///tmp/{self.spider_name}'
-        )
+        self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}")
 
     def test_empty_dict(self):
         def uri_params(params, spider):
             return {}
 
         settings = self.build_settings(
-            uri='file:///tmp/%(name)s',
+            uri="file:///tmp/%(name)s",
             uri_params=uri_params,
         )
         crawler, feed_exporter = self._crawler_feed_exporter(settings)
@@ -2675,7 +2785,7 @@ class URIParamsTest:
             return params
 
         settings = self.build_settings(
-            uri='file:///tmp/%(name)s',
+            uri="file:///tmp/%(name)s",
             uri_params=uri_params,
         )
         crawler, feed_exporter = self._crawler_feed_exporter(settings)
@@ -2685,17 +2795,14 @@ class URIParamsTest:
             warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
 
-        self.assertEqual(
-            feed_exporter.slots[0].uri,
-            f'file:///tmp/{self.spider_name}'
-        )
+        self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}")
 
     def test_custom_param(self):
         def uri_params(params, spider):
-            return {**params, 'foo': self.spider_name}
+            return {**params, "foo": self.spider_name}
 
         settings = self.build_settings(
-            uri='file:///tmp/%(foo)s',
+            uri="file:///tmp/%(foo)s",
             uri_params=uri_params,
         )
         crawler, feed_exporter = self._crawler_feed_exporter(settings)
@@ -2705,21 +2812,18 @@ class URIParamsTest:
             warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
 
-        self.assertEqual(
-            feed_exporter.slots[0].uri,
-            f'file:///tmp/{self.spider_name}'
-        )
+        self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}")
 
 
 class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
     deprecated_options = True
 
-    def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
+    def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
         extra_settings = {}
         if uri_params:
-            extra_settings['FEED_URI_PARAMS'] = uri_params
+            extra_settings["FEED_URI_PARAMS"] = uri_params
         return {
-            'FEED_URI': uri,
+            "FEED_URI": uri,
             **extra_settings,
         }
 
@@ -2727,14 +2831,14 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
 class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
     deprecated_options = False
 
-    def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
+    def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
         options = {
-            'format': 'jl',
+            "format": "jl",
         }
         if uri_params:
-            options['uri_params'] = uri_params
+            options["uri_params"] = uri_params
         return {
-            'FEEDS': {
+            "FEEDS": {
                 uri: options,
             },
         }
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index e461bcf2c..88345d2bc 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -9,7 +9,12 @@ from unittest import mock, skipIf
 from urllib.parse import urlencode
 
 from twisted.internet import reactor
-from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
+from twisted.internet.defer import (
+    CancelledError,
+    Deferred,
+    DeferredList,
+    inlineCallbacks,
+)
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
@@ -27,22 +32,19 @@ from tests.mockserver import ssl_context_factory, LeafResource, Status
 
 
 def generate_random_string(size):
-    return ''.join(random.choices(
-        string.ascii_uppercase + string.digits,
-        k=size
-    ))
+    return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))
 
 
 def make_html_body(val):
-    response = f'''<html>
+    response = f"""<html>
 <h1>Hello from HTTP2<h1>
 <p>{val}</p>
-</html>'''
-    return bytes(response, 'utf-8')
+</html>"""
+    return bytes(response, "utf-8")
 
 
 class DummySpider(Spider):
-    name = 'dummy'
+    name = "dummy"
     start_urls: list = []
 
     def parse(self, response):
@@ -51,33 +53,33 @@ class DummySpider(Spider):
 
 class Data:
     SMALL_SIZE = 1024  # 1 KB
-    LARGE_SIZE = 1024 ** 2  # 1 MB
+    LARGE_SIZE = 1024**2  # 1 MB
 
     STR_SMALL = generate_random_string(SMALL_SIZE)
     STR_LARGE = generate_random_string(LARGE_SIZE)
 
     EXTRA_SMALL = generate_random_string(1024 * 15)
-    EXTRA_LARGE = generate_random_string((1024 ** 2) * 15)
+    EXTRA_LARGE = generate_random_string((1024**2) * 15)
 
     HTML_SMALL = make_html_body(STR_SMALL)
     HTML_LARGE = make_html_body(STR_LARGE)
 
-    JSON_SMALL = {'data': STR_SMALL}
-    JSON_LARGE = {'data': STR_LARGE}
+    JSON_SMALL = {"data": STR_SMALL}
+    JSON_LARGE = {"data": STR_LARGE}
 
-    DATALOSS = b'Dataloss Content'
-    NO_CONTENT_LENGTH = b'This response do not have any content-length header'
+    DATALOSS = b"Dataloss Content"
+    NO_CONTENT_LENGTH = b"This response do not have any content-length header"
 
 
 class GetDataHtmlSmall(LeafResource):
     def render_GET(self, request: TxRequest):
-        request.setHeader('Content-Type', 'text/html; charset=UTF-8')
+        request.setHeader("Content-Type", "text/html; charset=UTF-8")
         return Data.HTML_SMALL
 
 
 class GetDataHtmlLarge(LeafResource):
     def render_GET(self, request: TxRequest):
-        request.setHeader('Content-Type', 'text/html; charset=UTF-8')
+        request.setHeader("Content-Type", "text/html; charset=UTF-8")
         return Data.HTML_LARGE
 
 
@@ -85,16 +87,16 @@ class PostDataJsonMixin:
     @staticmethod
     def make_response(request: TxRequest, extra_data: str):
         response = {
-            'request-headers': {},
-            'request-body': json.loads(request.content.read()),
-            'extra-data': extra_data
+            "request-headers": {},
+            "request-body": json.loads(request.content.read()),
+            "extra-data": extra_data,
         }
         for k, v in request.requestHeaders.getAllRawHeaders():
-            response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8')
+            response["request-headers"][str(k, "utf-8")] = str(v[0], "utf-8")
 
-        response_bytes = bytes(json.dumps(response), 'utf-8')
-        request.setHeader('Content-Type', 'application/json; charset=UTF-8')
-        request.setHeader('Content-Encoding', 'UTF-8')
+        response_bytes = bytes(json.dumps(response), "utf-8")
+        request.setHeader("Content-Type", "application/json; charset=UTF-8")
+        request.setHeader("Content-Encoding", "UTF-8")
         return response_bytes
 
 
@@ -109,7 +111,6 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin):
 
 
 class Dataloss(LeafResource):
-
     def render_GET(self, request: TxRequest):
         request.setHeader(b"Content-Length", b"1024")
         self.deferRequest(request, 0, self._delayed_render, request)
@@ -123,7 +124,7 @@ class Dataloss(LeafResource):
 
 class NoContentLengthHeader(LeafResource):
     def render_GET(self, request: TxRequest):
-        request.requestHeaders.removeHeader('Content-Length')
+        request.requestHeaders.removeHeader("Content-Length")
         self.deferRequest(request, 0, self._delayed_render, request)
         return NOT_DONE_YET
 
@@ -140,57 +141,61 @@ class TimeoutResponse(LeafResource):
 
 class QueryParams(LeafResource):
     def render_GET(self, request: TxRequest):
-        request.setHeader('Content-Type', 'application/json; charset=UTF-8')
-        request.setHeader('Content-Encoding', 'UTF-8')
+        request.setHeader("Content-Type", "application/json; charset=UTF-8")
+        request.setHeader("Content-Encoding", "UTF-8")
 
         query_params = {}
         for k, v in request.args.items():
-            query_params[str(k, 'utf-8')] = str(v[0], 'utf-8')
+            query_params[str(k, "utf-8")] = str(v[0], "utf-8")
 
-        return bytes(json.dumps(query_params), 'utf-8')
+        return bytes(json.dumps(query_params), "utf-8")
 
 
 class RequestHeaders(LeafResource):
     """Sends all the headers received as a response"""
 
     def render_GET(self, request: TxRequest):
-        request.setHeader('Content-Type', 'application/json; charset=UTF-8')
-        request.setHeader('Content-Encoding', 'UTF-8')
+        request.setHeader("Content-Type", "application/json; charset=UTF-8")
+        request.setHeader("Content-Encoding", "UTF-8")
         headers = {}
         for k, v in request.requestHeaders.getAllRawHeaders():
-            headers[str(k, 'utf-8')] = str(v[0], 'utf-8')
+            headers[str(k, "utf-8")] = str(v[0], "utf-8")
 
-        return bytes(json.dumps(headers), 'utf-8')
+        return bytes(json.dumps(headers), "utf-8")
 
 
-def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate:
-    pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8")
+def get_client_certificate(
+    key_file: Path, certificate_file: Path
+) -> PrivateCertificate:
+    pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(
+        encoding="utf-8"
+    )
 
     return PrivateCertificate.loadPEM(pem)
 
 
 @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ClientProtocolTestCase(TestCase):
-    scheme = 'https'
-    key_file = Path(__file__).parent / 'keys' / 'localhost.key'
-    certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt'
+    scheme = "https"
+    key_file = Path(__file__).parent / "keys" / "localhost.key"
+    certificate_file = Path(__file__).parent / "keys" / "localhost.crt"
 
     def _init_resource(self):
         self.temp_directory = self.mktemp()
         Path(self.temp_directory).mkdir()
         r = File(self.temp_directory)
-        r.putChild(b'get-data-html-small', GetDataHtmlSmall())
-        r.putChild(b'get-data-html-large', GetDataHtmlLarge())
+        r.putChild(b"get-data-html-small", GetDataHtmlSmall())
+        r.putChild(b"get-data-html-large", GetDataHtmlLarge())
 
-        r.putChild(b'post-data-json-small', PostDataJsonSmall())
-        r.putChild(b'post-data-json-large', PostDataJsonLarge())
+        r.putChild(b"post-data-json-small", PostDataJsonSmall())
+        r.putChild(b"post-data-json-large", PostDataJsonLarge())
 
-        r.putChild(b'dataloss', Dataloss())
-        r.putChild(b'no-content-length-header', NoContentLengthHeader())
-        r.putChild(b'status', Status())
-        r.putChild(b'query-params', QueryParams())
-        r.putChild(b'timeout', TimeoutResponse())
-        r.putChild(b'request-headers', RequestHeaders())
+        r.putChild(b"dataloss", Dataloss())
+        r.putChild(b"no-content-length-header", NoContentLengthHeader())
+        r.putChild(b"status", Status())
+        r.putChild(b"query-params", QueryParams())
+        r.putChild(b"timeout", TimeoutResponse())
+        r.putChild(b"request-headers", RequestHeaders())
         return r
 
     @inlineCallbacks
@@ -200,26 +205,35 @@ class Https2ClientProtocolTestCase(TestCase):
         self.site = Site(root, timeout=None)
 
         # Start server for testing
-        self.hostname = 'localhost'
-        context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file))
+        self.hostname = "localhost"
+        context_factory = ssl_context_factory(
+            str(self.key_file), str(self.certificate_file)
+        )
 
-        server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
+        server_endpoint = SSL4ServerEndpoint(
+            reactor, 0, context_factory, interface=self.hostname
+        )
         self.server = yield server_endpoint.listen(self.site)
         self.port_number = self.server.getHost().port
 
         # Connect H2 client with server
-        self.client_certificate = get_client_certificate(self.key_file, self.certificate_file)
+        self.client_certificate = get_client_certificate(
+            self.key_file, self.certificate_file
+        )
         client_options = optionsForClientTLS(
             hostname=self.hostname,
             trustRoot=self.client_certificate,
-            acceptableProtocols=[b'h2']
+            acceptableProtocols=[b"h2"],
         )
-        uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
+        uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8"))
 
         self.conn_closed_deferred = Deferred()
         from scrapy.core.http2.protocol import H2ClientFactory
+
         h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
-        client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
+        client_endpoint = SSL4ClientEndpoint(
+            reactor, self.hostname, self.port_number, client_options
+        )
         self.client = yield client_endpoint.connect(h2_client_factory)
 
     @inlineCallbacks
@@ -236,8 +250,8 @@ class Https2ClientProtocolTestCase(TestCase):
         :param path: Should have / at the starting compulsorily if not empty
         :return: Complete url
         """
-        assert len(path) > 0 and (path[0] == '/' or path[0] == '&')
-        return f'{self.scheme}://{self.hostname}:{self.port_number}{path}'
+        assert len(path) > 0 and (path[0] == "/" or path[0] == "&")
+        return f"{self.scheme}://{self.hostname}:{self.port_number}{path}"
 
     def make_request(self, request: Request) -> Deferred:
         return self.client.request(request, DummySpider())
@@ -251,18 +265,13 @@ class Https2ClientProtocolTestCase(TestCase):
 
         return DeferredList(d_list, fireOnOneErrback=True)
 
-    def _check_GET(
-        self,
-        request: Request,
-        expected_body,
-        expected_status
-    ):
+    def _check_GET(self, request: Request, expected_body, expected_status):
         def check_response(response: Response):
             self.assertEqual(response.status, expected_status)
             self.assertEqual(response.body, expected_body)
             self.assertEqual(response.request, request)
 
-            content_length = int(response.headers.get('Content-Length'))
+            content_length = int(response.headers.get("Content-Length"))
             self.assertEqual(len(response.body), content_length)
 
         d = self.make_request(request)
@@ -271,11 +280,11 @@ class Https2ClientProtocolTestCase(TestCase):
         return d
 
     def test_GET_small_body(self):
-        request = Request(self.get_url('/get-data-html-small'))
+        request = Request(self.get_url("/get-data-html-small"))
         return self._check_GET(request, Data.HTML_SMALL, 200)
 
     def test_GET_large_body(self):
-        request = Request(self.get_url('/get-data-html-large'))
+        request = Request(self.get_url("/get-data-html-large"))
         return self._check_GET(request, Data.HTML_LARGE, 200)
 
     def _check_GET_x10(self, *args, **kwargs):
@@ -286,16 +295,12 @@ class Https2ClientProtocolTestCase(TestCase):
 
     def test_GET_small_body_x10(self):
         return self._check_GET_x10(
-            Request(self.get_url('/get-data-html-small')),
-            Data.HTML_SMALL,
-            200
+            Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200
         )
 
     def test_GET_large_body_x10(self):
         return self._check_GET_x10(
-            Request(self.get_url('/get-data-html-large')),
-            Data.HTML_LARGE,
-            200
+            Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200
         )
 
     def _check_POST_json(
@@ -303,7 +308,7 @@ class Https2ClientProtocolTestCase(TestCase):
         request: Request,
         expected_request_body,
         expected_extra_data,
-        expected_status: int
+        expected_status: int,
     ):
         d = self.make_request(request)
 
@@ -311,50 +316,48 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(response.status, expected_status)
             self.assertEqual(response.request, request)
 
-            content_length = int(response.headers.get('Content-Length'))
+            content_length = int(response.headers.get("Content-Length"))
             self.assertEqual(len(response.body), content_length)
 
             # Parse the body
-            content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8')
+            content_encoding = str(response.headers[b"Content-Encoding"], "utf-8")
             body = json.loads(str(response.body, content_encoding))
-            self.assertIn('request-body', body)
-            self.assertIn('extra-data', body)
-            self.assertIn('request-headers', body)
+            self.assertIn("request-body", body)
+            self.assertIn("extra-data", body)
+            self.assertIn("request-headers", body)
 
-            request_body = body['request-body']
+            request_body = body["request-body"]
             self.assertEqual(request_body, expected_request_body)
 
-            extra_data = body['extra-data']
+            extra_data = body["extra-data"]
             self.assertEqual(extra_data, expected_extra_data)
 
             # Check if headers were sent successfully
-            request_headers = body['request-headers']
+            request_headers = body["request-headers"]
             for k, v in request.headers.items():
-                k_str = str(k, 'utf-8')
+                k_str = str(k, "utf-8")
                 self.assertIn(k_str, request_headers)
-                self.assertEqual(request_headers[k_str], str(v[0], 'utf-8'))
+                self.assertEqual(request_headers[k_str], str(v[0], "utf-8"))
 
         d.addCallback(assert_response)
         d.addErrback(self.fail)
         return d
 
     def test_POST_small_json(self):
-        request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL)
-        return self._check_POST_json(
-            request,
-            Data.JSON_SMALL,
-            Data.EXTRA_SMALL,
-            200
+        request = JsonRequest(
+            url=self.get_url("/post-data-json-small"),
+            method="POST",
+            data=Data.JSON_SMALL,
         )
+        return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200)
 
     def test_POST_large_json(self):
-        request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE)
-        return self._check_POST_json(
-            request,
-            Data.JSON_LARGE,
-            Data.EXTRA_LARGE,
-            200
+        request = JsonRequest(
+            url=self.get_url("/post-data-json-large"),
+            method="POST",
+            data=Data.JSON_LARGE,
         )
+        return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200)
 
     def _check_POST_json_x10(self, *args, **kwargs):
         def get_deferred():
@@ -363,32 +366,36 @@ class Https2ClientProtocolTestCase(TestCase):
         return self._check_repeat(get_deferred, 10)
 
     def test_POST_small_json_x10(self):
-        request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL)
+        request = JsonRequest(
+            url=self.get_url("/post-data-json-small"),
+            method="POST",
+            data=Data.JSON_SMALL,
+        )
         return self._check_POST_json_x10(
-            request,
-            Data.JSON_SMALL,
-            Data.EXTRA_SMALL,
-            200
+            request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200
         )
 
     def test_POST_large_json_x10(self):
-        request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE)
+        request = JsonRequest(
+            url=self.get_url("/post-data-json-large"),
+            method="POST",
+            data=Data.JSON_LARGE,
+        )
         return self._check_POST_json_x10(
-            request,
-            Data.JSON_LARGE,
-            Data.EXTRA_LARGE,
-            200
+            request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200
         )
 
     @inlineCallbacks
     def test_invalid_negotiated_protocol(self):
-        with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"):
-            request = Request(url=self.get_url('/status?n=200'))
+        with mock.patch(
+            "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
+        ):
+            request = Request(url=self.get_url("/status?n=200"))
             with self.assertRaises(ResponseFailed):
                 yield self.make_request(request)
 
     def test_cancel_request(self):
-        request = Request(url=self.get_url('/get-data-html-large'))
+        request = Request(url=self.get_url("/get-data-html-large"))
 
         def assert_response(response: Response):
             self.assertEqual(response.status, 499)
@@ -402,13 +409,15 @@ class Https2ClientProtocolTestCase(TestCase):
         return d
 
     def test_download_maxsize_exceeded(self):
-        request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000})
+        request = Request(
+            url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000}
+        )
 
         def assert_cancelled_error(failure):
             self.assertIsInstance(failure.value, CancelledError)
             error_pattern = re.compile(
-                rf'Cancelling download of {request.url}: received response '
-                rf'size \(\d*\) larger than download max size \(1000\)'
+                rf"Cancelling download of {request.url}: received response "
+                rf"size \(\d*\) larger than download max size \(1000\)"
             )
             self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1)
 
@@ -420,15 +429,18 @@ class Https2ClientProtocolTestCase(TestCase):
     def test_received_dataloss_response(self):
         """In case when value of Header Content-Length != len(Received Data)
         ProtocolError is raised"""
-        request = Request(url=self.get_url('/dataloss'))
+        request = Request(url=self.get_url("/dataloss"))
 
         def assert_failure(failure: Failure):
             self.assertTrue(len(failure.value.reasons) > 0)
             from h2.exceptions import InvalidBodyLengthError
-            self.assertTrue(any(
-                isinstance(error, InvalidBodyLengthError)
-                for error in failure.value.reasons
-            ))
+
+            self.assertTrue(
+                any(
+                    isinstance(error, InvalidBodyLengthError)
+                    for error in failure.value.reasons
+                )
+            )
 
         d = self.make_request(request)
         d.addCallback(self.fail)
@@ -436,13 +448,13 @@ class Https2ClientProtocolTestCase(TestCase):
         return d
 
     def test_missing_content_length_header(self):
-        request = Request(url=self.get_url('/no-content-length-header'))
+        request = Request(url=self.get_url("/no-content-length-header"))
 
         def assert_content_length(response: Response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.body, Data.NO_CONTENT_LENGTH)
             self.assertEqual(response.request, request)
-            self.assertNotIn('Content-Length', response.headers)
+            self.assertNotIn("Content-Length", response.headers)
 
         d = self.make_request(request)
         d.addCallback(assert_content_length)
@@ -450,40 +462,39 @@ class Https2ClientProtocolTestCase(TestCase):
         return d
 
     @inlineCallbacks
-    def _check_log_warnsize(
-        self,
-        request,
-        warn_pattern,
-        expected_body
-    ):
-        with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm:
+    def _check_log_warnsize(self, request, warn_pattern, expected_body):
+        with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm:
             response = yield self.make_request(request)
             self.assertEqual(response.status, 200)
             self.assertEqual(response.request, request)
             self.assertEqual(response.body, expected_body)
 
             # Check the warning is raised only once for this request
-            self.assertEqual(sum(
-                len(re.findall(warn_pattern, log))
-                for log in cm.output
-            ), 1)
+            self.assertEqual(
+                sum(len(re.findall(warn_pattern, log)) for log in cm.output), 1
+            )
 
     @inlineCallbacks
     def test_log_expected_warnsize(self):
-        request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000})
+        request = Request(
+            url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000}
+        )
         warn_pattern = re.compile(
-            rf'Expected response size \(\d*\) larger than '
-            rf'download warn size \(1000\) in request {request}'
+            rf"Expected response size \(\d*\) larger than "
+            rf"download warn size \(1000\) in request {request}"
         )
 
         yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)
 
     @inlineCallbacks
     def test_log_received_warnsize(self):
-        request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10})
+        request = Request(
+            url=self.get_url("/no-content-length-header"),
+            meta={"download_warnsize": 10},
+        )
         warn_pattern = re.compile(
-            rf'Received more \(\d*\) bytes than download '
-            rf'warn size \(10\) in request {request}'
+            rf"Received more \(\d*\) bytes than download "
+            rf"warn size \(10\) in request {request}"
         )
 
         yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
@@ -495,9 +506,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def get_deferred():
             return self._check_GET(
-                Request(self.get_url('/get-data-html-small')),
-                Data.HTML_SMALL,
-                200
+                Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200
             )
 
         return self._check_repeat(get_deferred, 500)
@@ -511,20 +520,20 @@ class Https2ClientProtocolTestCase(TestCase):
         def assert_inactive_stream(failure):
             self.assertIsNotNone(failure.check(ResponseFailed))
             from scrapy.core.http2.stream import InactiveStreamClosed
-            self.assertTrue(any(
-                isinstance(e, InactiveStreamClosed)
-                for e in failure.value.reasons
-            ))
+
+            self.assertTrue(
+                any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons)
+            )
 
         # Send 100 request (we do not check the result)
         for _ in range(100):
-            d = self.make_request(Request(self.get_url('/get-data-html-small')))
+            d = self.make_request(Request(self.get_url("/get-data-html-small")))
             d.addBoth(lambda _: None)
             d_list.append(d)
 
         # Now send 10 extra request and save the response deferred in a list
         for _ in range(10):
-            d = self.make_request(Request(self.get_url('/get-data-html-small')))
+            d = self.make_request(Request(self.get_url("/get-data-html-small")))
             d.addCallback(self.fail)
             d.addErrback(assert_inactive_stream)
             d_list.append(d)
@@ -537,19 +546,19 @@ class Https2ClientProtocolTestCase(TestCase):
 
     def test_invalid_request_type(self):
         with self.assertRaises(TypeError):
-            self.make_request('https://InvalidDataTypePassed.com')
+            self.make_request("https://InvalidDataTypePassed.com")
 
     def test_query_parameters(self):
         params = {
-            'a': generate_random_string(20),
-            'b': generate_random_string(20),
-            'c': generate_random_string(20),
-            'd': generate_random_string(20)
+            "a": generate_random_string(20),
+            "b": generate_random_string(20),
+            "c": generate_random_string(20),
+            "d": generate_random_string(20),
         }
-        request = Request(self.get_url(f'/query-params?{urlencode(params)}'))
+        request = Request(self.get_url(f"/query-params?{urlencode(params)}"))
 
         def assert_query_params(response: Response):
-            content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8')
+            content_encoding = str(response.headers[b"Content-Encoding"], "utf-8")
             data = json.loads(str(response.body, content_encoding))
             self.assertEqual(data, params)
 
@@ -565,7 +574,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         d_list = []
         for status in [200, 404]:
-            request = Request(self.get_url(f'/status?n={status}'))
+            request = Request(self.get_url(f"/status?n={status}"))
             d = self.make_request(request)
             d.addCallback(assert_response_status, status)
             d.addErrback(self.fail)
@@ -574,17 +583,23 @@ class Https2ClientProtocolTestCase(TestCase):
         return DeferredList(d_list, fireOnOneErrback=True)
 
     def test_response_has_correct_certificate_ip_address(self):
-        request = Request(self.get_url('/status?n=200'))
+        request = Request(self.get_url("/status?n=200"))
 
         def assert_metadata(response: Response):
             self.assertEqual(response.request, request)
             self.assertIsInstance(response.certificate, Certificate)
             self.assertIsNotNone(response.certificate.original)
-            self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer())
-            self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey()))
+            self.assertEqual(
+                response.certificate.getIssuer(), self.client_certificate.getIssuer()
+            )
+            self.assertTrue(
+                response.certificate.getPublicKey().matches(
+                    self.client_certificate.getPublicKey()
+                )
+            )
 
             self.assertIsInstance(response.ip_address, IPv4Address)
-            self.assertEqual(str(response.ip_address), '127.0.0.1')
+            self.assertEqual(str(response.ip_address), "127.0.0.1")
 
         d = self.make_request(request)
         d.addCallback(assert_metadata)
@@ -597,10 +612,11 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_invalid_hostname(failure: Failure):
             from scrapy.core.http2.stream import InvalidHostname
+
             self.assertIsNotNone(failure.check(InvalidHostname))
             error_msg = str(failure.value)
-            self.assertIn('localhost', error_msg)
-            self.assertIn('127.0.0.1', error_msg)
+            self.assertIn("localhost", error_msg)
+            self.assertIn("127.0.0.1", error_msg)
             self.assertIn(str(request), error_msg)
 
         d = self.make_request(request)
@@ -609,24 +625,24 @@ class Https2ClientProtocolTestCase(TestCase):
         return d
 
     def test_invalid_hostname(self):
-        return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain')
+        return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain")
 
     def test_invalid_host_port(self):
         port = self.port_number + 1
-        return self._check_invalid_netloc(f'https://127.0.0.1:{port}')
+        return self._check_invalid_netloc(f"https://127.0.0.1:{port}")
 
     def test_connection_stays_with_invalid_requests(self):
         d_list = [
             self.test_invalid_hostname(),
             self.test_invalid_host_port(),
             self.test_GET_small_body(),
-            self.test_POST_small_json()
+            self.test_POST_small_json(),
         ]
 
         return DeferredList(d_list, fireOnOneErrback=True)
 
     def test_connection_timeout(self):
-        request = Request(self.get_url('/timeout'))
+        request = Request(self.get_url("/timeout"))
         d = self.make_request(request)
 
         # Update the timer to 1s to test connection timeout
@@ -635,8 +651,12 @@ class Https2ClientProtocolTestCase(TestCase):
         def assert_timeout_error(failure: Failure):
             for err in failure.value.reasons:
                 from scrapy.core.http2.protocol import H2ClientProtocol
+
                 if isinstance(err, TimeoutError):
-                    self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
+                    self.assertIn(
+                        f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s",
+                        str(err),
+                    )
                     break
             else:
                 self.fail()
@@ -646,20 +666,20 @@ class Https2ClientProtocolTestCase(TestCase):
         return d
 
     def test_request_headers_received(self):
-        request = Request(self.get_url('/request-headers'), headers={
-            'header-1': 'header value 1',
-            'header-2': 'header value 2'
-        })
+        request = Request(
+            self.get_url("/request-headers"),
+            headers={"header-1": "header value 1", "header-2": "header value 2"},
+        )
         d = self.make_request(request)
 
         def assert_request_headers(response: Response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.request, request)
 
-            response_headers = json.loads(str(response.body, 'utf-8'))
+            response_headers = json.loads(str(response.body, "utf-8"))
             self.assertIsInstance(response_headers, dict)
             for k, v in request.headers.items():
-                k, v = str(k, 'utf-8'), str(v[0], 'utf-8')
+                k, v = str(k, "utf-8"), str(v[0], "utf-8")
                 self.assertIn(k, response_headers)
                 self.assertEqual(v, response_headers[k])
 
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 08420332c..ea42cadcd 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -6,10 +6,10 @@ from scrapy.http.cookies import WrappedRequest, WrappedResponse
 
 
 class WrappedRequestTest(TestCase):
-
     def setUp(self):
-        self.request = Request("http://www.example.com/page.html",
-                               headers={"Content-Type": "text/html"})
+        self.request = Request(
+            "http://www.example.com/page.html", headers={"Content-Type": "text/html"}
+        )
         self.wrapped = WrappedRequest(self.request)
 
     def test_get_full_url(self):
@@ -29,35 +29,34 @@ class WrappedRequestTest(TestCase):
         self.assertFalse(self.wrapped.unverifiable)
 
     def test_is_unverifiable2(self):
-        self.request.meta['is_unverifiable'] = True
+        self.request.meta["is_unverifiable"] = True
         self.assertTrue(self.wrapped.is_unverifiable())
         self.assertTrue(self.wrapped.unverifiable)
 
     def test_get_origin_req_host(self):
-        self.assertEqual(self.wrapped.origin_req_host, 'www.example.com')
+        self.assertEqual(self.wrapped.origin_req_host, "www.example.com")
 
     def test_has_header(self):
-        self.assertTrue(self.wrapped.has_header('content-type'))
-        self.assertFalse(self.wrapped.has_header('xxxxx'))
+        self.assertTrue(self.wrapped.has_header("content-type"))
+        self.assertFalse(self.wrapped.has_header("xxxxx"))
 
     def test_get_header(self):
-        self.assertEqual(self.wrapped.get_header('content-type'), 'text/html')
-        self.assertEqual(self.wrapped.get_header('xxxxx', 'def'), 'def')
+        self.assertEqual(self.wrapped.get_header("content-type"), "text/html")
+        self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def")
 
     def test_header_items(self):
-        self.assertEqual(self.wrapped.header_items(),
-                         [('Content-Type', ['text/html'])])
+        self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])])
 
     def test_add_unredirected_header(self):
-        self.wrapped.add_unredirected_header('hello', 'world')
-        self.assertEqual(self.request.headers['hello'], b'world')
+        self.wrapped.add_unredirected_header("hello", "world")
+        self.assertEqual(self.request.headers["hello"], b"world")
 
 
 class WrappedResponseTest(TestCase):
-
     def setUp(self):
-        self.response = Response("http://www.example.com/page.html",
-                                 headers={"Content-TYpe": "text/html"})
+        self.response = Response(
+            "http://www.example.com/page.html", headers={"Content-TYpe": "text/html"}
+        )
         self.wrapped = WrappedResponse(self.response)
 
     def test_info(self):
@@ -65,4 +64,4 @@ class WrappedResponseTest(TestCase):
 
     def test_get_all(self):
         # get_all result must be native string
-        self.assertEqual(self.wrapped.get_all('content-type'), ['text/html'])
+        self.assertEqual(self.wrapped.get_all("content-type"), ["text/html"])
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 1ca936247..566bb302d 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -5,38 +5,39 @@ from scrapy.http import Headers
 
 
 class HeadersTest(unittest.TestCase):
-
     def assertSortedEqual(self, first, second, msg=None):
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_basics(self):
-        h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234})
-        assert h['Content-Type']
-        assert h['Content-Length']
+        h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
+        assert h["Content-Type"]
+        assert h["Content-Length"]
 
-        self.assertRaises(KeyError, h.__getitem__, 'Accept')
-        self.assertEqual(h.get('Accept'), None)
-        self.assertEqual(h.getlist('Accept'), [])
+        self.assertRaises(KeyError, h.__getitem__, "Accept")
+        self.assertEqual(h.get("Accept"), None)
+        self.assertEqual(h.getlist("Accept"), [])
 
-        self.assertEqual(h.get('Accept', '*/*'), b'*/*')
-        self.assertEqual(h.getlist('Accept', '*/*'), [b'*/*'])
-        self.assertEqual(h.getlist('Accept', ['text/html', 'images/jpeg']),
-                         [b'text/html', b'images/jpeg'])
+        self.assertEqual(h.get("Accept", "*/*"), b"*/*")
+        self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"])
+        self.assertEqual(
+            h.getlist("Accept", ["text/html", "images/jpeg"]),
+            [b"text/html", b"images/jpeg"],
+        )
 
     def test_single_value(self):
         h = Headers()
-        h['Content-Type'] = 'text/html'
-        self.assertEqual(h['Content-Type'], b'text/html')
-        self.assertEqual(h.get('Content-Type'), b'text/html')
-        self.assertEqual(h.getlist('Content-Type'), [b'text/html'])
+        h["Content-Type"] = "text/html"
+        self.assertEqual(h["Content-Type"], b"text/html")
+        self.assertEqual(h.get("Content-Type"), b"text/html")
+        self.assertEqual(h.getlist("Content-Type"), [b"text/html"])
 
     def test_multivalue(self):
         h = Headers()
-        h['X-Forwarded-For'] = hlist = ['ip1', 'ip2']
-        self.assertEqual(h['X-Forwarded-For'], b'ip2')
-        self.assertEqual(h.get('X-Forwarded-For'), b'ip2')
-        self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
-        assert h.getlist('X-Forwarded-For') is not hlist
+        h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]
+        self.assertEqual(h["X-Forwarded-For"], b"ip2")
+        self.assertEqual(h.get("X-Forwarded-For"), b"ip2")
+        self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"])
+        assert h.getlist("X-Forwarded-For") is not hlist
 
     def test_multivalue_for_one_header(self):
         h = Headers((("a", "b"), ("a", "c")))
@@ -45,117 +46,121 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h.getlist("a"), [b"b", b"c"])
 
     def test_encode_utf8(self):
-        h = Headers({'key': '\xa3'}, encoding='utf-8')
+        h = Headers({"key": "\xa3"}, encoding="utf-8")
         key, val = dict(h).popitem()
         assert isinstance(key, bytes), key
         assert isinstance(val[0], bytes), val[0]
-        self.assertEqual(val[0], b'\xc2\xa3')
+        self.assertEqual(val[0], b"\xc2\xa3")
 
     def test_encode_latin1(self):
-        h = Headers({'key': '\xa3'}, encoding='latin1')
+        h = Headers({"key": "\xa3"}, encoding="latin1")
         key, val = dict(h).popitem()
-        self.assertEqual(val[0], b'\xa3')
+        self.assertEqual(val[0], b"\xa3")
 
     def test_encode_multiple(self):
-        h = Headers({'key': ['\xa3']}, encoding='utf-8')
+        h = Headers({"key": ["\xa3"]}, encoding="utf-8")
         key, val = dict(h).popitem()
-        self.assertEqual(val[0], b'\xc2\xa3')
+        self.assertEqual(val[0], b"\xc2\xa3")
 
     def test_delete_and_contains(self):
         h = Headers()
-        h['Content-Type'] = 'text/html'
-        assert 'Content-Type' in h
-        del h['Content-Type']
-        assert 'Content-Type' not in h
+        h["Content-Type"] = "text/html"
+        assert "Content-Type" in h
+        del h["Content-Type"]
+        assert "Content-Type" not in h
 
     def test_setdefault(self):
         h = Headers()
-        hlist = ['ip1', 'ip2']
-        olist = h.setdefault('X-Forwarded-For', hlist)
-        assert h.getlist('X-Forwarded-For') is not hlist
-        assert h.getlist('X-Forwarded-For') is olist
+        hlist = ["ip1", "ip2"]
+        olist = h.setdefault("X-Forwarded-For", hlist)
+        assert h.getlist("X-Forwarded-For") is not hlist
+        assert h.getlist("X-Forwarded-For") is olist
 
         h = Headers()
-        olist = h.setdefault('X-Forwarded-For', 'ip1')
-        self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1'])
-        assert h.getlist('X-Forwarded-For') is olist
+        olist = h.setdefault("X-Forwarded-For", "ip1")
+        self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"])
+        assert h.getlist("X-Forwarded-For") is olist
 
     def test_iterables(self):
-        idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']}
+        idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
 
         h = Headers(idict)
-        self.assertDictEqual(dict(h),
-                             {b'Content-Type': [b'text/html'],
-                              b'X-Forwarded-For': [b'ip1', b'ip2']})
-        self.assertSortedEqual(h.keys(),
-                               [b'X-Forwarded-For', b'Content-Type'])
-        self.assertSortedEqual(h.items(),
-                               [(b'X-Forwarded-For', [b'ip1', b'ip2']),
-                                (b'Content-Type', [b'text/html'])])
-        self.assertSortedEqual(h.values(), [b'ip2', b'text/html'])
+        self.assertDictEqual(
+            dict(h),
+            {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]},
+        )
+        self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"])
+        self.assertSortedEqual(
+            h.items(),
+            [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])],
+        )
+        self.assertSortedEqual(h.values(), [b"ip2", b"text/html"])
 
     def test_update(self):
         h = Headers()
-        h.update({'Content-Type': 'text/html',
-                  'X-Forwarded-For': ['ip1', 'ip2']})
-        self.assertEqual(h.getlist('Content-Type'), [b'text/html'])
-        self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
+        h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
+        self.assertEqual(h.getlist("Content-Type"), [b"text/html"])
+        self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"])
 
     def test_copy(self):
-        h1 = Headers({'header1': ['value1', 'value2']})
+        h1 = Headers({"header1": ["value1", "value2"]})
         h2 = copy.copy(h1)
         self.assertEqual(h1, h2)
-        self.assertEqual(h1.getlist('header1'), h2.getlist('header1'))
-        assert h1.getlist('header1') is not h2.getlist('header1')
+        self.assertEqual(h1.getlist("header1"), h2.getlist("header1"))
+        assert h1.getlist("header1") is not h2.getlist("header1")
         assert isinstance(h2, Headers)
 
     def test_appendlist(self):
-        h1 = Headers({'header1': 'value1'})
-        h1.appendlist('header1', 'value3')
-        self.assertEqual(h1.getlist('header1'), [b'value1', b'value3'])
+        h1 = Headers({"header1": "value1"})
+        h1.appendlist("header1", "value3")
+        self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"])
 
         h1 = Headers()
-        h1.appendlist('header1', 'value1')
-        h1.appendlist('header1', 'value3')
-        self.assertEqual(h1.getlist('header1'), [b'value1', b'value3'])
+        h1.appendlist("header1", "value1")
+        h1.appendlist("header1", "value3")
+        self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"])
 
     def test_setlist(self):
-        h1 = Headers({'header1': 'value1'})
-        self.assertEqual(h1.getlist('header1'), [b'value1'])
-        h1.setlist('header1', [b'value2', b'value3'])
-        self.assertEqual(h1.getlist('header1'), [b'value2', b'value3'])
+        h1 = Headers({"header1": "value1"})
+        self.assertEqual(h1.getlist("header1"), [b"value1"])
+        h1.setlist("header1", [b"value2", b"value3"])
+        self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"])
 
     def test_setlistdefault(self):
-        h1 = Headers({'header1': 'value1'})
-        h1.setlistdefault('header1', ['value2', 'value3'])
-        h1.setlistdefault('header2', ['value2', 'value3'])
-        self.assertEqual(h1.getlist('header1'), [b'value1'])
-        self.assertEqual(h1.getlist('header2'), [b'value2', b'value3'])
+        h1 = Headers({"header1": "value1"})
+        h1.setlistdefault("header1", ["value2", "value3"])
+        h1.setlistdefault("header2", ["value2", "value3"])
+        self.assertEqual(h1.getlist("header1"), [b"value1"])
+        self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"])
 
     def test_none_value(self):
         h1 = Headers()
-        h1['foo'] = 'bar'
-        h1['foo'] = None
-        h1.setdefault('foo', 'bar')
-        self.assertEqual(h1.get('foo'), None)
-        self.assertEqual(h1.getlist('foo'), [])
+        h1["foo"] = "bar"
+        h1["foo"] = None
+        h1.setdefault("foo", "bar")
+        self.assertEqual(h1.get("foo"), None)
+        self.assertEqual(h1.getlist("foo"), [])
 
     def test_int_value(self):
-        h1 = Headers({'hey': 5})
-        h1['foo'] = 1
-        h1.setdefault('bar', 2)
-        h1.setlist('buz', [1, 'dos', 3])
-        self.assertEqual(h1.getlist('foo'), [b'1'])
-        self.assertEqual(h1.getlist('bar'), [b'2'])
-        self.assertEqual(h1.getlist('buz'), [b'1', b'dos', b'3'])
-        self.assertEqual(h1.getlist('hey'), [b'5'])
+        h1 = Headers({"hey": 5})
+        h1["foo"] = 1
+        h1.setdefault("bar", 2)
+        h1.setlist("buz", [1, "dos", 3])
+        self.assertEqual(h1.getlist("foo"), [b"1"])
+        self.assertEqual(h1.getlist("bar"), [b"2"])
+        self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"])
+        self.assertEqual(h1.getlist("hey"), [b"5"])
 
     def test_invalid_value(self):
-        self.assertRaisesRegex(TypeError, 'Unsupported value type',
-                               Headers, {'foo': object()})
-        self.assertRaisesRegex(TypeError, 'Unsupported value type',
-                               Headers().__setitem__, 'foo', object())
-        self.assertRaisesRegex(TypeError, 'Unsupported value type',
-                               Headers().setdefault, 'foo', object())
-        self.assertRaisesRegex(TypeError, 'Unsupported value type',
-                               Headers().setlist, 'foo', [object()])
+        self.assertRaisesRegex(
+            TypeError, "Unsupported value type", Headers, {"foo": object()}
+        )
+        self.assertRaisesRegex(
+            TypeError, "Unsupported value type", Headers().__setitem__, "foo", object()
+        )
+        self.assertRaisesRegex(
+            TypeError, "Unsupported value type", Headers().setdefault, "foo", object()
+        )
+        self.assertRaisesRegex(
+            TypeError, "Unsupported value type", Headers().setlist, "foo", [object()]
+        )
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9f7f1854f..b6f999b29 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -6,14 +6,21 @@ import warnings
 from unittest import mock
 from urllib.parse import parse_qs, unquote_to_bytes, urlparse
 
-from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
+from scrapy.http import (
+    Request,
+    FormRequest,
+    XmlRpcRequest,
+    JsonRequest,
+    Headers,
+    HtmlResponse,
+)
 from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
 
     request_class = Request
-    default_method = 'GET'
+    default_method = "GET"
     default_headers = {}
     default_meta = {}
 
@@ -23,7 +30,7 @@ class RequestTest(unittest.TestCase):
 
         # url argument must be basestring
         self.assertRaises(TypeError, self.request_class, 123)
-        r = self.request_class('http://www.example.com')
+        r = self.request_class("http://www.example.com")
 
         r = self.request_class("http://www.example.com")
         assert isinstance(r.url, str)
@@ -36,7 +43,9 @@ class RequestTest(unittest.TestCase):
 
         meta = {"lala": "lolo"}
         headers = {b"caca": b"coco"}
-        r = self.request_class("http://www.example.com", meta=meta, headers=headers, body="a body")
+        r = self.request_class(
+            "http://www.example.com", meta=meta, headers=headers, body="a body"
+        )
 
         assert r.meta is not meta
         self.assertEqual(r.meta, meta)
@@ -45,22 +54,22 @@ class RequestTest(unittest.TestCase):
 
     def test_url_scheme(self):
         # This test passes by not raising any (ValueError) exception
-        self.request_class('http://example.org')
-        self.request_class('https://example.org')
-        self.request_class('s3://example.org')
-        self.request_class('ftp://example.org')
-        self.request_class('about:config')
-        self.request_class('data:,Hello%2C%20World!')
+        self.request_class("http://example.org")
+        self.request_class("https://example.org")
+        self.request_class("s3://example.org")
+        self.request_class("ftp://example.org")
+        self.request_class("about:config")
+        self.request_class("data:,Hello%2C%20World!")
 
     def test_url_no_scheme(self):
-        self.assertRaises(ValueError, self.request_class, 'foo')
-        self.assertRaises(ValueError, self.request_class, '/foo/')
-        self.assertRaises(ValueError, self.request_class, '/foo:bar')
+        self.assertRaises(ValueError, self.request_class, "foo")
+        self.assertRaises(ValueError, self.request_class, "/foo/")
+        self.assertRaises(ValueError, self.request_class, "/foo:bar")
 
     def test_headers(self):
         # Different ways of setting headers attribute
-        url = 'http://www.scrapy.org'
-        headers = {b'Accept': 'gzip', b'Custom-Header': 'nothing to tell you'}
+        url = "http://www.scrapy.org"
+        headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
         r = self.request_class(url=url, headers=headers)
         p = self.request_class(url=url, headers=r.headers)
 
@@ -69,15 +78,15 @@ class RequestTest(unittest.TestCase):
         self.assertFalse(p.headers is r.headers)
 
         # headers must not be unicode
-        h = Headers({'key1': 'val1', 'key2': 'val2'})
-        h['newkey'] = 'newval'
+        h = Headers({"key1": "val1", "key2": "val2"})
+        h["newkey"] = "newval"
         for k, v in h.items():
             self.assertIsInstance(k, bytes)
             for s in v:
                 self.assertIsInstance(s, bytes)
 
     def test_eq(self):
-        url = 'http://www.scrapy.org'
+        url = "http://www.scrapy.org"
         r1 = self.request_class(url=url)
         r2 = self.request_class(url=url)
         self.assertNotEqual(r1, r2)
@@ -115,13 +124,17 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
         # should be same as above
-        r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
+        r2 = self.request_class(
+            url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
+        )
         self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
     def test_url_encoding_query_latin1(self):
         # encoding is used for encoding query-string before percent-escaping;
         # path is still UTF-8 encoded before percent-escaping
-        r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
+        r3 = self.request_class(
+            url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
+        )
         self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
 
     def test_url_encoding_nonutf8_untouched(self):
@@ -154,27 +167,35 @@ class RequestTest(unittest.TestCase):
 
     def test_body(self):
         r1 = self.request_class(url="http://www.example.com/")
-        assert r1.body == b''
+        assert r1.body == b""
 
         r2 = self.request_class(url="http://www.example.com/", body=b"")
         assert isinstance(r2.body, bytes)
-        self.assertEqual(r2.encoding, 'utf-8')  # default encoding
+        self.assertEqual(r2.encoding, "utf-8")  # default encoding
 
-        r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
+        r3 = self.request_class(
+            url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
+        )
         assert isinstance(r3.body, bytes)
         self.assertEqual(r3.body, b"Price: \xc2\xa3100")
 
-        r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
+        r4 = self.request_class(
+            url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
+        )
         assert isinstance(r4.body, bytes)
         self.assertEqual(r4.body, b"Price: \xa3100")
 
     def test_ajax_url(self):
         # ascii url
         r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
-        self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
+        self.assertEqual(
+            r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue"
+        )
         # unicode url
         r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
-        self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
+        self.assertEqual(
+            r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue"
+        )
 
     def test_copy(self):
         """Test Request copy"""
@@ -182,10 +203,14 @@ class RequestTest(unittest.TestCase):
         def somecallback():
             pass
 
-        r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
-                                callback=somecallback, errback=somecallback)
-        r1.meta['foo'] = 'bar'
-        r1.cb_kwargs['key'] = 'value'
+        r1 = self.request_class(
+            "http://www.example.com",
+            flags=["f1", "f2"],
+            callback=somecallback,
+            errback=somecallback,
+        )
+        r1.meta["foo"] = "bar"
+        r1.cb_kwargs["key"] = "value"
         r2 = r1.copy()
 
         # make sure copy does not propagate callbacks
@@ -199,7 +224,9 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r1.flags, r2.flags)
 
         # make sure cb_kwargs dict is shallow copied
-        assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical"
+        assert (
+            r1.cb_kwargs is not r2.cb_kwargs
+        ), "cb_kwargs must be a shallow copy, not identical"
         self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
 
         # make sure meta dict is shallow copied
@@ -207,7 +234,9 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r1.meta, r2.meta)
 
         # make sure headers attribute is shallow copied
-        assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical"
+        assert (
+            r1.headers is not r2.headers
+        ), "headers must be a shallow copy, not identical"
         self.assertEqual(r1.headers, r2.headers)
         self.assertEqual(r1.encoding, r2.encoding)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
@@ -220,27 +249,31 @@ class RequestTest(unittest.TestCase):
         class CustomRequest(self.request_class):
             pass
 
-        r1 = CustomRequest('http://www.example.com')
+        r1 = CustomRequest("http://www.example.com")
         r2 = r1.copy()
 
         assert isinstance(r2, CustomRequest)
 
     def test_replace(self):
         """Test Request.replace() method"""
-        r1 = self.request_class("http://www.example.com", method='GET')
+        r1 = self.request_class("http://www.example.com", method="GET")
         hdrs = Headers(r1.headers)
-        hdrs[b'key'] = b'value'
+        hdrs[b"key"] = b"value"
         r2 = r1.replace(method="POST", body="New body", headers=hdrs)
         self.assertEqual(r1.url, r2.url)
         self.assertEqual((r1.method, r2.method), ("GET", "POST"))
-        self.assertEqual((r1.body, r2.body), (b'', b"New body"))
+        self.assertEqual((r1.body, r2.body), (b"", b"New body"))
         self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs))
 
         # Empty attributes (which may fail if not compared properly)
-        r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True)
-        r4 = r3.replace(url="http://www.example.com/2", body=b'', meta={}, dont_filter=False)
+        r3 = self.request_class(
+            "http://www.example.com", meta={"a": 1}, dont_filter=True
+        )
+        r4 = r3.replace(
+            url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
+        )
         self.assertEqual(r4.url, "http://www.example.com/2")
-        self.assertEqual(r4.body, b'')
+        self.assertEqual(r4.body, b"")
         self.assertEqual(r4.meta, {})
         assert r4.dont_filter is False
 
@@ -250,27 +283,27 @@ class RequestTest(unittest.TestCase):
 
     def test_immutable_attributes(self):
         r = self.request_class("http://example.com")
-        self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
-        self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
+        self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
+        self.assertRaises(AttributeError, setattr, r, "body", "xxx")
 
     def test_callback_and_errback(self):
         def a_function():
             pass
 
-        r1 = self.request_class('http://example.com')
+        r1 = self.request_class("http://example.com")
         self.assertIsNone(r1.callback)
         self.assertIsNone(r1.errback)
 
-        r2 = self.request_class('http://example.com', callback=a_function)
+        r2 = self.request_class("http://example.com", callback=a_function)
         self.assertIs(r2.callback, a_function)
         self.assertIsNone(r2.errback)
 
-        r3 = self.request_class('http://example.com', errback=a_function)
+        r3 = self.request_class("http://example.com", errback=a_function)
         self.assertIsNone(r3.callback)
         self.assertIs(r3.errback, a_function)
 
         r4 = self.request_class(
-            url='http://example.com',
+            url="http://example.com",
             callback=a_function,
             errback=a_function,
         )
@@ -279,14 +312,14 @@ class RequestTest(unittest.TestCase):
 
     def test_callback_and_errback_type(self):
         with self.assertRaises(TypeError):
-            self.request_class('http://example.com', callback='a_function')
+            self.request_class("http://example.com", callback="a_function")
         with self.assertRaises(TypeError):
-            self.request_class('http://example.com', errback='a_function')
+            self.request_class("http://example.com", errback="a_function")
         with self.assertRaises(TypeError):
             self.request_class(
-                url='http://example.com',
-                callback='a_function',
-                errback='a_function',
+                url="http://example.com",
+                callback="a_function",
+                errback="a_function",
             )
 
     def test_from_curl(self):
@@ -311,37 +344,48 @@ class RequestTest(unittest.TestCase):
         r = self.request_class.from_curl(curl_command)
         self.assertEqual(r.method, "POST")
         self.assertEqual(r.url, "http://httpbin.org/post")
-        self.assertEqual(r.body,
-                         b"custname=John+Smith&custtel=500&custemail=jsmith%40"
-                         b"example.org&size=small&topping=cheese&topping=onion"
-                         b"&delivery=12%3A15&comments=")
-        self.assertEqual(r.cookies, {
-            '_gauges_unique_year': '1',
-            '_gauges_unique': '1',
-            '_gauges_unique_month': '1',
-            '_gauges_unique_hour': '1',
-            '_gauges_unique_day': '1'
-        })
-        self.assertEqual(r.headers, {
-            b'Origin': [b'http://httpbin.org'],
-            b'Accept-Encoding': [b'gzip, deflate'],
-            b'Accept-Language': [b'en-US,en;q=0.9,ru;q=0.8,es;q=0.7'],
-            b'Upgrade-Insecure-Requests': [b'1'],
-            b'User-Agent': [b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.'
-                            b'36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202'
-                            b'.75 Chrome/62.0.3202.75 Safari/537.36'],
-            b'Content-Type': [b'application /x-www-form-urlencoded'],
-            b'Accept': [b'text/html,application/xhtml+xml,application/xml;q=0.'
-                        b'9,image/webp,image/apng,*/*;q=0.8'],
-            b'Cache-Control': [b'max-age=0'],
-            b'Referer': [b'http://httpbin.org/forms/post'],
-            b'Connection': [b'keep-alive']})
+        self.assertEqual(
+            r.body,
+            b"custname=John+Smith&custtel=500&custemail=jsmith%40"
+            b"example.org&size=small&topping=cheese&topping=onion"
+            b"&delivery=12%3A15&comments=",
+        )
+        self.assertEqual(
+            r.cookies,
+            {
+                "_gauges_unique_year": "1",
+                "_gauges_unique": "1",
+                "_gauges_unique_month": "1",
+                "_gauges_unique_hour": "1",
+                "_gauges_unique_day": "1",
+            },
+        )
+        self.assertEqual(
+            r.headers,
+            {
+                b"Origin": [b"http://httpbin.org"],
+                b"Accept-Encoding": [b"gzip, deflate"],
+                b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
+                b"Upgrade-Insecure-Requests": [b"1"],
+                b"User-Agent": [
+                    b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
+                    b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
+                    b".75 Chrome/62.0.3202.75 Safari/537.36"
+                ],
+                b"Content-Type": [b"application /x-www-form-urlencoded"],
+                b"Accept": [
+                    b"text/html,application/xhtml+xml,application/xml;q=0."
+                    b"9,image/webp,image/apng,*/*;q=0.8"
+                ],
+                b"Cache-Control": [b"max-age=0"],
+                b"Referer": [b"http://httpbin.org/forms/post"],
+                b"Connection": [b"keep-alive"],
+            },
+        )
 
     def test_from_curl_with_kwargs(self):
         r = self.request_class.from_curl(
-            'curl -X PATCH "http://example.org"',
-            method="POST",
-            meta={'key': 'value'}
+            'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
         )
         self.assertEqual(r.method, "POST")
         self.assertEqual(r.meta, {"key": "value"})
@@ -349,7 +393,7 @@ class RequestTest(unittest.TestCase):
     def test_from_curl_ignore_unknown_options(self):
         # By default: it works and ignores the unknown options: --foo and -z
         with warnings.catch_warnings():  # avoid warning when executing tests
-            warnings.simplefilter('ignore')
+            warnings.simplefilter("ignore")
             r = self.request_class.from_curl(
                 'curl -X DELETE "http://example.org" --foo -z',
             )
@@ -377,68 +421,85 @@ class FormRequestTest(RequestTest):
 
     def test_empty_formdata(self):
         r1 = self.request_class("http://www.example.com", formdata={})
-        self.assertEqual(r1.body, b'')
+        self.assertEqual(r1.body, b"")
 
     def test_formdata_overrides_querystring(self):
-        data = (('a', 'one'), ('a', 'two'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment',
-                                 method='GET', formdata=data).url.split('#')[0]
-        fs = _qs(self.request_class(url, method='GET', formdata=data))
-        self.assertEqual(set(fs[b'a']), {b'one', b'two'})
-        self.assertEqual(fs[b'b'], [b'2'])
-        self.assertIsNone(fs.get(b'c'))
+        data = (("a", "one"), ("a", "two"), ("b", "2"))
+        url = self.request_class(
+            "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data
+        ).url.split("#")[0]
+        fs = _qs(self.request_class(url, method="GET", formdata=data))
+        self.assertEqual(set(fs[b"a"]), {b"one", b"two"})
+        self.assertEqual(fs[b"b"], [b"2"])
+        self.assertIsNone(fs.get(b"c"))
 
-        data = {'a': '1', 'b': '2'}
-        fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
+        data = {"a": "1", "b": "2"}
+        fs = _qs(
+            self.request_class("http://www.example.com/", method="GET", formdata=data)
+        )
+        self.assertEqual(fs[b"a"], [b"1"])
+        self.assertEqual(fs[b"b"], [b"2"])
 
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
-        data = {b'one': b'two', b'price': b'\xc2\xa3 100'}
+        data = {b"one": b"two", b"price": b"\xc2\xa3 100"}
         r2 = self.request_class("http://www.example.com", formdata=data)
-        self.assertEqual(r2.method, 'POST')
-        self.assertEqual(r2.encoding, 'utf-8')
-        self.assertQueryEqual(r2.body, b'price=%C2%A3+100&one=two')
-        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(r2.method, "POST")
+        self.assertEqual(r2.encoding, "utf-8")
+        self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two")
+        self.assertEqual(
+            r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded"
+        )
 
     def test_default_encoding_textual_data(self):
         # using default encoding (utf-8)
-        data = {'µ one': 'two', 'price': '£ 100'}
+        data = {"µ one": "two", "price": "£ 100"}
         r2 = self.request_class("http://www.example.com", formdata=data)
-        self.assertEqual(r2.method, 'POST')
-        self.assertEqual(r2.encoding, 'utf-8')
-        self.assertQueryEqual(r2.body, b'price=%C2%A3+100&%C2%B5+one=two')
-        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(r2.method, "POST")
+        self.assertEqual(r2.encoding, "utf-8")
+        self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
+        self.assertEqual(
+            r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded"
+        )
 
     def test_default_encoding_mixed_data(self):
         # using default encoding (utf-8)
-        data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
+        data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
         r2 = self.request_class("http://www.example.com", formdata=data)
-        self.assertEqual(r2.method, 'POST')
-        self.assertEqual(r2.encoding, 'utf-8')
-        self.assertQueryEqual(r2.body, b'%C2%B5one=two&price%C2%A3=%C2%A3+100')
-        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(r2.method, "POST")
+        self.assertEqual(r2.encoding, "utf-8")
+        self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
+        self.assertEqual(
+            r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded"
+        )
 
     def test_custom_encoding_bytes(self):
-        data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
-        r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
-        self.assertEqual(r2.method, 'POST')
-        self.assertEqual(r2.encoding, 'latin1')
-        self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
-        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+        data = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
+        r2 = self.request_class(
+            "http://www.example.com", formdata=data, encoding="latin1"
+        )
+        self.assertEqual(r2.method, "POST")
+        self.assertEqual(r2.encoding, "latin1")
+        self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two")
+        self.assertEqual(
+            r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded"
+        )
 
     def test_custom_encoding_textual_data(self):
-        data = {'price': '£ 100'}
-        r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
-        self.assertEqual(r3.encoding, 'latin1')
-        self.assertEqual(r3.body, b'price=%A3+100')
+        data = {"price": "£ 100"}
+        r3 = self.request_class(
+            "http://www.example.com", formdata=data, encoding="latin1"
+        )
+        self.assertEqual(r3.encoding, "latin1")
+        self.assertEqual(r3.body, b"price=%A3+100")
 
     def test_multi_key_values(self):
         # using multiples values for a single key
-        data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
+        data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]}
         r3 = self.request_class("http://www.example.com", formdata=data)
-        self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
+        self.assertQueryEqual(
+            r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100"
+        )
 
     def test_from_response_post(self):
         response = _buildresponse(
@@ -447,17 +508,22 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
             </form>""",
-            url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
+            url="http://www.example.com/this/list.html",
+        )
+        req = self.request_class.from_response(
+            response, formdata={"one": ["two", "three"], "six": "seven"}
+        )
 
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers[b"Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(set(fs[b'test']), {b'val1', b'val2'})
-        self.assertEqual(set(fs[b'one']), {b'two', b'three'})
-        self.assertEqual(fs[b'test2'], [b'xxx'])
-        self.assertEqual(fs[b'six'], [b'seven'])
+        self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"})
+        self.assertEqual(set(fs[b"one"]), {b"two", b"three"})
+        self.assertEqual(fs[b"test2"], [b"xxx"])
+        self.assertEqual(fs[b"six"], [b"seven"])
 
     def test_from_response_post_nonascii_bytes_utf8(self):
         response = _buildresponse(
@@ -466,17 +532,22 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test \xc2\xa3" value="val2">
             <input type="hidden" name="test2" value="xxx \xc2\xb5">
             </form>""",
-            url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
+            url="http://www.example.com/this/list.html",
+        )
+        req = self.request_class.from_response(
+            response, formdata={"one": ["two", "three"], "six": "seven"}
+        )
 
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers[b"Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
-        self.assertEqual(set(fs['one']), {'two', 'three'})
-        self.assertEqual(fs['test2'], ['xxx µ'])
-        self.assertEqual(fs['six'], ['seven'])
+        self.assertEqual(set(fs["test £"]), {"val1", "val2"})
+        self.assertEqual(set(fs["one"]), {"two", "three"})
+        self.assertEqual(fs["test2"], ["xxx µ"])
+        self.assertEqual(fs["six"], ["seven"])
 
     def test_from_response_post_nonascii_bytes_latin1(self):
         response = _buildresponse(
@@ -486,18 +557,22 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx \xb5">
             </form>""",
             url="http://www.example.com/this/list.html",
-            encoding='latin1',
+            encoding="latin1",
+        )
+        req = self.request_class.from_response(
+            response, formdata={"one": ["two", "three"], "six": "seven"}
         )
-        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers[b"Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
-        fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
-        self.assertEqual(set(fs['one']), {'two', 'three'})
-        self.assertEqual(fs['test2'], ['xxx µ'])
-        self.assertEqual(fs['six'], ['seven'])
+        fs = _qs(req, to_unicode=True, encoding="latin1")
+        self.assertEqual(set(fs["test £"]), {"val1", "val2"})
+        self.assertEqual(set(fs["one"]), {"two", "three"})
+        self.assertEqual(fs["test2"], ["xxx µ"])
+        self.assertEqual(fs["six"], ["seven"])
 
     def test_from_response_post_nonascii_unicode(self):
         response = _buildresponse(
@@ -506,42 +581,46 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test £" value="val2">
             <input type="hidden" name="test2" value="xxx µ">
             </form>""",
-            url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
+            url="http://www.example.com/this/list.html",
+        )
+        req = self.request_class.from_response(
+            response, formdata={"one": ["two", "three"], "six": "seven"}
+        )
 
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers[b"Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
-        self.assertEqual(set(fs['one']), {'two', 'three'})
-        self.assertEqual(fs['test2'], ['xxx µ'])
-        self.assertEqual(fs['six'], ['seven'])
+        self.assertEqual(set(fs["test £"]), {"val1", "val2"})
+        self.assertEqual(set(fs["one"]), {"two", "three"})
+        self.assertEqual(fs["test2"], ["xxx µ"])
+        self.assertEqual(fs["six"], ["seven"])
 
     def test_from_response_duplicate_form_key(self):
-        response = _buildresponse(
-            '<form></form>',
-            url='http://www.example.com')
+        response = _buildresponse("<form></form>", url="http://www.example.com")
         req = self.request_class.from_response(
             response=response,
-            method='GET',
-            formdata=(('foo', 'bar'), ('foo', 'baz')),
+            method="GET",
+            formdata=(("foo", "bar"), ("foo", "baz")),
         )
-        self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
-        self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
+        self.assertEqual(urlparse(req.url).hostname, "www.example.com")
+        self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz")
 
     def test_from_response_override_duplicate_form_key(self):
         response = _buildresponse(
             """<form action="get.php" method="POST">
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(
-            response,
-            formdata=(('two', '2'), ('two', '4')))
+            response, formdata=(("two", "2"), ("two", "4"))
+        )
         fs = _qs(req)
-        self.assertEqual(fs[b'one'], [b'1'])
-        self.assertEqual(fs[b'two'], [b'2', b'4'])
+        self.assertEqual(fs[b"one"], [b"1"])
+        self.assertEqual(fs[b"two"], [b"2", b"4"])
 
     def test_from_response_extra_headers(self):
         response = _buildresponse(
@@ -549,15 +628,18 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val1">
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(
             response=response,
-            formdata={'one': ['two', 'three'], 'six': 'seven'},
+            formdata={"one": ["two", "three"], "six": "seven"},
             headers={"Accept-Encoding": "gzip,deflate"},
         )
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
-        self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers["Content-type"], b"application/x-www-form-urlencoded"
+        )
+        self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate")
 
     def test_from_response_get(self):
         response = _buildresponse(
@@ -566,60 +648,67 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
             </form>""",
-            url="http://www.example.com/this/list.html")
-        r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
-        self.assertEqual(r1.method, 'GET')
+            url="http://www.example.com/this/list.html",
+        )
+        r1 = self.request_class.from_response(
+            response, formdata={"one": ["two", "three"], "six": "seven"}
+        )
+        self.assertEqual(r1.method, "GET")
         self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
         self.assertEqual(urlparse(r1.url).path, "/this/get.php")
         fs = _qs(r1)
-        self.assertEqual(set(fs[b'test']), {b'val1', b'val2'})
-        self.assertEqual(set(fs[b'one']), {b'two', b'three'})
-        self.assertEqual(fs[b'test2'], [b'xxx'])
-        self.assertEqual(fs[b'six'], [b'seven'])
+        self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"})
+        self.assertEqual(set(fs[b"one"]), {b"two", b"three"})
+        self.assertEqual(fs[b"test2"], [b"xxx"])
+        self.assertEqual(fs[b"six"], [b"seven"])
 
     def test_from_response_override_params(self):
         response = _buildresponse(
             """<form action="get.php" method="POST">
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
-            </form>""")
-        req = self.request_class.from_response(response, formdata={'two': '2'})
+            </form>"""
+        )
+        req = self.request_class.from_response(response, formdata={"two": "2"})
         fs = _qs(req)
-        self.assertEqual(fs[b'one'], [b'1'])
-        self.assertEqual(fs[b'two'], [b'2'])
+        self.assertEqual(fs[b"one"], [b"1"])
+        self.assertEqual(fs[b"two"], [b"2"])
 
     def test_from_response_drop_params(self):
         response = _buildresponse(
             """<form action="get.php" method="POST">
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
-            </form>""")
-        req = self.request_class.from_response(response, formdata={'two': None})
+            </form>"""
+        )
+        req = self.request_class.from_response(response, formdata={"two": None})
         fs = _qs(req)
-        self.assertEqual(fs[b'one'], [b'1'])
-        self.assertNotIn(b'two', fs)
+        self.assertEqual(fs[b"one"], [b"1"])
+        self.assertNotIn(b"two", fs)
 
     def test_from_response_override_method(self):
         response = _buildresponse(
-            '''<html><body>
+            """<html><body>
             <form action="/app"></form>
-            </body></html>''')
+            </body></html>"""
+        )
         request = FormRequest.from_response(response)
-        self.assertEqual(request.method, 'GET')
-        request = FormRequest.from_response(response, method='POST')
-        self.assertEqual(request.method, 'POST')
+        self.assertEqual(request.method, "GET")
+        request = FormRequest.from_response(response, method="POST")
+        self.assertEqual(request.method, "POST")
 
     def test_from_response_override_url(self):
         response = _buildresponse(
-            '''<html><body>
+            """<html><body>
             <form action="/app"></form>
-            </body></html>''')
+            </body></html>"""
+        )
         request = FormRequest.from_response(response)
-        self.assertEqual(request.url, 'http://example.com/app')
-        request = FormRequest.from_response(response, url='http://foo.bar/absolute')
-        self.assertEqual(request.url, 'http://foo.bar/absolute')
-        request = FormRequest.from_response(response, url='/relative')
-        self.assertEqual(request.url, 'http://example.com/relative')
+        self.assertEqual(request.url, "http://example.com/app")
+        request = FormRequest.from_response(response, url="http://foo.bar/absolute")
+        self.assertEqual(request.url, "http://foo.bar/absolute")
+        request = FormRequest.from_response(response, url="/relative")
+        self.assertEqual(request.url, "http://example.com/relative")
 
     def test_from_response_case_insensitive(self):
         response = _buildresponse(
@@ -627,12 +716,13 @@ class FormRequestTest(RequestTest):
             <input type="SuBmIt" name="clickable1" value="clicked1">
             <input type="iMaGe" name="i1" src="http://my.image.org/1.jpg">
             <input type="submit" name="clickable2" value="clicked2">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(response)
         fs = _qs(req)
-        self.assertEqual(fs[b'clickable1'], [b'clicked1'])
-        self.assertFalse(b'i1' in fs, fs)  # xpath in _get_inputs()
-        self.assertFalse(b'clickable2' in fs, fs)  # xpath in _get_clickable()
+        self.assertEqual(fs[b"clickable1"], [b"clicked1"])
+        self.assertFalse(b"i1" in fs, fs)  # xpath in _get_inputs()
+        self.assertFalse(b"clickable2" in fs, fs)  # xpath in _get_clickable()
 
     def test_from_response_submit_first_clickable(self):
         response = _buildresponse(
@@ -641,13 +731,14 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
             <input type="submit" name="clickable2" value="clicked2">
-            </form>""")
-        req = self.request_class.from_response(response, formdata={'two': '2'})
+            </form>"""
+        )
+        req = self.request_class.from_response(response, formdata={"two": "2"})
         fs = _qs(req)
-        self.assertEqual(fs[b'clickable1'], [b'clicked1'])
-        self.assertFalse(b'clickable2' in fs, fs)
-        self.assertEqual(fs[b'one'], [b'1'])
-        self.assertEqual(fs[b'two'], [b'2'])
+        self.assertEqual(fs[b"clickable1"], [b"clicked1"])
+        self.assertFalse(b"clickable2" in fs, fs)
+        self.assertEqual(fs[b"one"], [b"1"])
+        self.assertEqual(fs[b"two"], [b"2"])
 
     def test_from_response_submit_not_first_clickable(self):
         response = _buildresponse(
@@ -656,15 +747,16 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
             <input type="submit" name="clickable2" value="clicked2">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(
-            response, formdata={'two': '2'}, clickdata={'name': 'clickable2'}
+            response, formdata={"two": "2"}, clickdata={"name": "clickable2"}
         )
         fs = _qs(req)
-        self.assertEqual(fs[b'clickable2'], [b'clicked2'])
-        self.assertFalse(b'clickable1' in fs, fs)
-        self.assertEqual(fs[b'one'], [b'1'])
-        self.assertEqual(fs[b'two'], [b'2'])
+        self.assertEqual(fs[b"clickable2"], [b"clicked2"])
+        self.assertFalse(b"clickable1" in fs, fs)
+        self.assertEqual(fs[b"one"], [b"1"])
+        self.assertEqual(fs[b"two"], [b"2"])
 
     def test_from_response_dont_submit_image_as_input(self):
         response = _buildresponse(
@@ -672,10 +764,11 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="i1" value="i1v">
             <input type="image" name="i2" src="http://my.image.org/1.jpg">
             <input type="submit" name="i3" value="i3v">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v']})
+        self.assertEqual(fs, {b"i1": [b"i1v"]})
 
     def test_from_response_dont_submit_reset_as_input(self):
         response = _buildresponse(
@@ -684,20 +777,22 @@ class FormRequestTest(RequestTest):
             <input type="text" name="i2" value="i2v">
             <input type="reset" name="resetme">
             <input type="submit" name="i3" value="i3v">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
+        self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]})
 
     def test_from_response_clickdata_does_not_ignore_image(self):
         response = _buildresponse(
             """<form>
             <input type="text" name="i1" value="i1v">
             <input id="image" name="i2" type="image" value="i2v" alt="Login" src="http://my.image.org/1.jpg">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(response)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
+        self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]})
 
     def test_from_response_multiple_clickdata(self):
         response = _buildresponse(
@@ -706,14 +801,15 @@ class FormRequestTest(RequestTest):
             <input type="submit" name="clickable" value="clicked2">
             <input type="hidden" name="one" value="clicked1">
             <input type="hidden" name="two" value="clicked2">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(
-            response, clickdata={'name': 'clickable', 'value': 'clicked2'}
+            response, clickdata={"name": "clickable", "value": "clicked2"}
         )
         fs = _qs(req)
-        self.assertEqual(fs[b'clickable'], [b'clicked2'])
-        self.assertEqual(fs[b'one'], [b'clicked1'])
-        self.assertEqual(fs[b'two'], [b'clicked2'])
+        self.assertEqual(fs[b"clickable"], [b"clicked2"])
+        self.assertEqual(fs[b"one"], [b"clicked1"])
+        self.assertEqual(fs[b"two"], [b"clicked2"])
 
     def test_from_response_unicode_clickdata(self):
         response = _buildresponse(
@@ -722,12 +818,13 @@ class FormRequestTest(RequestTest):
             <input type="submit" name="price in \u20ac" value="\u20ac 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
             <input type="hidden" name="eurosign" value="\u20ac">
-            </form>""")
+            </form>"""
+        )
         req = self.request_class.from_response(
-            response, clickdata={'name': 'price in \u00a3'}
+            response, clickdata={"name": "price in \u00a3"}
         )
         fs = _qs(req, to_unicode=True)
-        self.assertTrue(fs['price in \u00a3'])
+        self.assertTrue(fs["price in \u00a3"])
 
     def test_from_response_unicode_clickdata_latin1(self):
         response = _buildresponse(
@@ -737,12 +834,13 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="poundsign" value="\u00a3">
             <input type="hidden" name="yensign" value="\u00a5">
             </form>""",
-            encoding='latin1')
-        req = self.request_class.from_response(
-            response, clickdata={'name': 'price in \u00a5'}
+            encoding="latin1",
         )
-        fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertTrue(fs['price in \u00a5'])
+        req = self.request_class.from_response(
+            response, clickdata={"name": "price in \u00a5"}
+        )
+        fs = _qs(req, to_unicode=True, encoding="latin1")
+        self.assertTrue(fs["price in \u00a5"])
 
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
@@ -754,22 +852,25 @@ class FormRequestTest(RequestTest):
             <input type="submit" name="clickable" value="clicked2">
             <input type="hidden" name="field2" value="value2">
             </form>
-            """)
+            """
+        )
         req = self.request_class.from_response(
-            response, formname='form2', clickdata={'name': 'clickable'}
+            response, formname="form2", clickdata={"name": "clickable"}
         )
         fs = _qs(req)
-        self.assertEqual(fs[b'clickable'], [b'clicked2'])
-        self.assertEqual(fs[b'field2'], [b'value2'])
-        self.assertFalse(b'field1' in fs, fs)
+        self.assertEqual(fs[b"clickable"], [b"clicked2"])
+        self.assertEqual(fs[b"field2"], [b"value2"])
+        self.assertFalse(b"field1" in fs, fs)
 
     def test_from_response_override_clickable(self):
-        response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''')
+        response = _buildresponse(
+            """<form><input type="submit" name="clickme" value="one"> </form>"""
+        )
         req = self.request_class.from_response(
-            response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}
+            response, formdata={"clickme": "two"}, clickdata={"name": "clickme"}
         )
         fs = _qs(req)
-        self.assertEqual(fs[b'clickme'], [b'two'])
+        self.assertEqual(fs[b"clickme"], [b"two"])
 
     def test_from_response_dont_click(self):
         response = _buildresponse(
@@ -778,11 +879,12 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
             <input type="submit" name="clickable2" value="clicked2">
-            </form>""")
+            </form>"""
+        )
         r1 = self.request_class.from_response(response, dont_click=True)
         fs = _qs(r1)
-        self.assertFalse(b'clickable1' in fs, fs)
-        self.assertFalse(b'clickable2' in fs, fs)
+        self.assertFalse(b"clickable1" in fs, fs)
+        self.assertFalse(b"clickable2" in fs, fs)
 
     def test_from_response_ambiguous_clickdata(self):
         response = _buildresponse(
@@ -792,17 +894,27 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="3">
             <input type="submit" name="clickable2" value="clicked2">
-            </form>""")
-        self.assertRaises(ValueError, self.request_class.from_response,
-                          response, clickdata={'type': 'submit'})
+            </form>"""
+        )
+        self.assertRaises(
+            ValueError,
+            self.request_class.from_response,
+            response,
+            clickdata={"type": "submit"},
+        )
 
     def test_from_response_non_matching_clickdata(self):
         response = _buildresponse(
             """<form>
             <input type="submit" name="clickable" value="clicked">
-            </form>""")
-        self.assertRaises(ValueError, self.request_class.from_response,
-                          response, clickdata={'nonexistent': 'notme'})
+            </form>"""
+        )
+        self.assertRaises(
+            ValueError,
+            self.request_class.from_response,
+            response,
+            clickdata={"nonexistent": "notme"},
+        )
 
     def test_from_response_nr_index_clickdata(self):
         response = _buildresponse(
@@ -810,32 +922,37 @@ class FormRequestTest(RequestTest):
             <input type="submit" name="clickable1" value="clicked1">
             <input type="submit" name="clickable2" value="clicked2">
             </form>
-            """)
-        req = self.request_class.from_response(response, clickdata={'nr': 1})
+            """
+        )
+        req = self.request_class.from_response(response, clickdata={"nr": 1})
         fs = _qs(req)
-        self.assertIn(b'clickable2', fs)
-        self.assertNotIn(b'clickable1', fs)
+        self.assertIn(b"clickable2", fs)
+        self.assertNotIn(b"clickable1", fs)
 
     def test_from_response_invalid_nr_index_clickdata(self):
         response = _buildresponse(
             """<form>
             <input type="submit" name="clickable" value="clicked">
             </form>
-            """)
-        self.assertRaises(ValueError, self.request_class.from_response,
-                          response, clickdata={'nr': 1})
+            """
+        )
+        self.assertRaises(
+            ValueError, self.request_class.from_response, response, clickdata={"nr": 1}
+        )
 
     def test_from_response_errors_noform(self):
         response = _buildresponse("""<html></html>""")
         self.assertRaises(ValueError, self.request_class.from_response, response)
 
     def test_from_response_invalid_html5(self):
-        response = _buildresponse("""<!DOCTYPE html><body></html><form>"""
-                                  """<input type="text" name="foo" value="xxx">"""
-                                  """</form></body></html>""")
-        req = self.request_class.from_response(response, formdata={'bar': 'buz'})
+        response = _buildresponse(
+            """<!DOCTYPE html><body></html><form>"""
+            """<input type="text" name="foo" value="xxx">"""
+            """</form></body></html>"""
+        )
+        req = self.request_class.from_response(response, formdata={"bar": "buz"})
         fs = _qs(req)
-        self.assertEqual(fs, {b'foo': [b'xxx'], b'bar': [b'buz']})
+        self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]})
 
     def test_from_response_errors_formnumber(self):
         response = _buildresponse(
@@ -843,20 +960,26 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val1">
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
-            </form>""")
-        self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1)
+            </form>"""
+        )
+        self.assertRaises(
+            IndexError, self.request_class.from_response, response, formnumber=1
+        )
 
     def test_from_response_noformname(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="2">
-            </form>""")
-        r1 = self.request_class.from_response(response, formdata={'two': '3'})
-        self.assertEqual(r1.method, 'POST')
-        self.assertEqual(r1.headers['Content-type'], b'application/x-www-form-urlencoded')
+            </form>"""
+        )
+        r1 = self.request_class.from_response(response, formdata={"two": "3"})
+        self.assertEqual(r1.method, "POST")
+        self.assertEqual(
+            r1.headers["Content-type"], b"application/x-www-form-urlencoded"
+        )
         fs = _qs(r1)
-        self.assertEqual(fs, {b'one': [b'1'], b'two': [b'3']})
+        self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]})
 
     def test_from_response_formname_exists(self):
         response = _buildresponse(
@@ -867,11 +990,12 @@ class FormRequestTest(RequestTest):
             <form name="form2" action="post.php" method="POST">
             <input type="hidden" name="three" value="3">
             <input type="hidden" name="four" value="4">
-            </form>""")
+            </form>"""
+        )
         r1 = self.request_class.from_response(response, formname="form2")
-        self.assertEqual(r1.method, 'POST')
+        self.assertEqual(r1.method, "POST")
         fs = _qs(r1)
-        self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
+        self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]})
 
     def test_from_response_formname_notexist(self):
         response = _buildresponse(
@@ -880,11 +1004,12 @@ class FormRequestTest(RequestTest):
             </form>
             <form name="form2" action="post.php" method="POST">
             <input type="hidden" name="two" value="2">
-            </form>""")
+            </form>"""
+        )
         r1 = self.request_class.from_response(response, formname="form3")
-        self.assertEqual(r1.method, 'POST')
+        self.assertEqual(r1.method, "POST")
         fs = _qs(r1)
-        self.assertEqual(fs, {b'one': [b'1']})
+        self.assertEqual(fs, {b"one": [b"1"]})
 
     def test_from_response_formname_errors_formnumber(self):
         response = _buildresponse(
@@ -893,9 +1018,15 @@ class FormRequestTest(RequestTest):
             </form>
             <form name="form2" action="post.php" method="POST">
             <input type="hidden" name="two" value="2">
-            </form>""")
-        self.assertRaises(IndexError, self.request_class.from_response,
-                          response, formname="form3", formnumber=2)
+            </form>"""
+        )
+        self.assertRaises(
+            IndexError,
+            self.request_class.from_response,
+            response,
+            formname="form3",
+            formnumber=2,
+        )
 
     def test_from_response_formid_exists(self):
         response = _buildresponse(
@@ -906,11 +1037,12 @@ class FormRequestTest(RequestTest):
             <form id="form2" action="post.php" method="POST">
             <input type="hidden" name="three" value="3">
             <input type="hidden" name="four" value="4">
-            </form>""")
+            </form>"""
+        )
         r1 = self.request_class.from_response(response, formid="form2")
-        self.assertEqual(r1.method, 'POST')
+        self.assertEqual(r1.method, "POST")
         fs = _qs(r1)
-        self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
+        self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]})
 
     def test_from_response_formname_notexists_fallback_formid(self):
         response = _buildresponse(
@@ -921,11 +1053,14 @@ class FormRequestTest(RequestTest):
             <form id="form2" name="form2" action="post.php" method="POST">
             <input type="hidden" name="three" value="3">
             <input type="hidden" name="four" value="4">
-            </form>""")
-        r1 = self.request_class.from_response(response, formname="form3", formid="form2")
-        self.assertEqual(r1.method, 'POST')
+            </form>"""
+        )
+        r1 = self.request_class.from_response(
+            response, formname="form3", formid="form2"
+        )
+        self.assertEqual(r1.method, "POST")
         fs = _qs(r1)
-        self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
+        self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]})
 
     def test_from_response_formid_notexist(self):
         response = _buildresponse(
@@ -934,11 +1069,12 @@ class FormRequestTest(RequestTest):
             </form>
             <form id="form2" action="post.php" method="POST">
             <input type="hidden" name="two" value="2">
-            </form>""")
+            </form>"""
+        )
         r1 = self.request_class.from_response(response, formid="form3")
-        self.assertEqual(r1.method, 'POST')
+        self.assertEqual(r1.method, "POST")
         fs = _qs(r1)
-        self.assertEqual(fs, {b'one': [b'1']})
+        self.assertEqual(fs, {b"one": [b"1"]})
 
     def test_from_response_formid_errors_formnumber(self):
         response = _buildresponse(
@@ -947,13 +1083,19 @@ class FormRequestTest(RequestTest):
             </form>
             <form id="form2" name="form2" action="post.php" method="POST">
             <input type="hidden" name="two" value="2">
-            </form>""")
-        self.assertRaises(IndexError, self.request_class.from_response,
-                          response, formid="form3", formnumber=2)
+            </form>"""
+        )
+        self.assertRaises(
+            IndexError,
+            self.request_class.from_response,
+            response,
+            formid="form3",
+            formnumber=2,
+        )
 
     def test_from_response_select(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <select name="i1">
                 <option value="i1v1">option 1</option>
                 <option value="i1v2" selected>option 2</option>
@@ -977,14 +1119,15 @@ class FormRequestTest(RequestTest):
             </select>
             <select name="i6"></select>
             <select name="i7"/>
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(fs, {'i1': ['i1v2'], 'i2': ['i2v1'], 'i4': ['i4v2', 'i4v3']})
+        self.assertEqual(fs, {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]})
 
     def test_from_response_radio(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <input type="radio" name="i1" value="i1v1">
             <input type="radio" name="i1" value="iv2" checked>
             <input type="radio" name="i2" checked>
@@ -993,14 +1136,15 @@ class FormRequestTest(RequestTest):
             <input type="radio" name="i3">
             <input type="radio" value="i4v1">
             <input type="radio">
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']})
+        self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]})
 
     def test_from_response_checkbox(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <input type="checkbox" name="i1" value="i1v1">
             <input type="checkbox" name="i1" value="iv2" checked>
             <input type="checkbox" name="i2" checked>
@@ -1009,51 +1153,55 @@ class FormRequestTest(RequestTest):
             <input type="checkbox" name="i3">
             <input type="checkbox" value="i4v1">
             <input type="checkbox">
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']})
+        self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]})
 
     def test_from_response_input_text(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <input type="text" name="i1" value="i1v1">
             <input type="text" name="i2">
             <input type="text" value="i3v1">
             <input type="text">
             <input name="i4" value="i4v1">
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b''], b'i4': [b'i4v1']})
+        self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]})
 
     def test_from_response_input_hidden(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <input type="hidden" name="i1" value="i1v1">
             <input type="hidden" name="i2">
             <input type="hidden" value="i3v1">
             <input type="hidden">
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b'']})
+        self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]})
 
     def test_from_response_input_textarea(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <textarea name="i1">i1v</textarea>
             <textarea name="i2"></textarea>
             <textarea name="i3"/>
             <textarea>i4v</textarea>
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b''], b'i3': [b'']})
+        self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]})
 
     def test_from_response_descendants(self):
         res = _buildresponse(
-            '''<form>
+            """<form>
             <div>
               <fieldset>
                 <input type="text" name="i1">
@@ -1067,10 +1215,11 @@ class FormRequestTest(RequestTest):
               <input type="hidden" name="h1" value="h1v">
               </div>
             <input type="hidden" name="h2" value="h2v">
-            </form>''')
+            </form>"""
+        )
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'})
+        self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"})
 
     def test_from_response_xpath(self):
         response = _buildresponse(
@@ -1081,28 +1230,43 @@ class FormRequestTest(RequestTest):
             <form action="post2.php" method="POST">
             <input type="hidden" name="three" value="3">
             <input type="hidden" name="four" value="4">
-            </form>""")
-        r1 = self.request_class.from_response(response, formxpath="//form[@action='post.php']")
+            </form>"""
+        )
+        r1 = self.request_class.from_response(
+            response, formxpath="//form[@action='post.php']"
+        )
         fs = _qs(r1)
-        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b"one"], [b"1"])
 
-        r1 = self.request_class.from_response(response, formxpath="//form/input[@name='four']")
+        r1 = self.request_class.from_response(
+            response, formxpath="//form/input[@name='four']"
+        )
         fs = _qs(r1)
-        self.assertEqual(fs[b'three'], [b'3'])
+        self.assertEqual(fs[b"three"], [b"3"])
 
-        self.assertRaises(ValueError, self.request_class.from_response,
-                          response, formxpath="//form/input[@name='abc']")
+        self.assertRaises(
+            ValueError,
+            self.request_class.from_response,
+            response,
+            formxpath="//form/input[@name='abc']",
+        )
 
     def test_from_response_unicode_xpath(self):
         response = _buildresponse(b'<form name="\xd1\x8a"></form>')
-        r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
+        r = self.request_class.from_response(
+            response, formxpath="//form[@name='\u044a']"
+        )
         fs = _qs(r)
         self.assertEqual(fs, {})
 
         xpath = "//form[@name='\u03b1']"
-        self.assertRaisesRegex(ValueError, re.escape(xpath),
-                               self.request_class.from_response,
-                               response, formxpath=xpath)
+        self.assertRaisesRegex(
+            ValueError,
+            re.escape(xpath),
+            self.request_class.from_response,
+            response,
+            formxpath=xpath,
+        )
 
     def test_from_response_button_submit(self):
         response = _buildresponse(
@@ -1111,15 +1275,18 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="val2">
             <button type="submit" name="button1" value="submit1">Submit</button>
             </form>""",
-            url="http://www.example.com/this/list.html")
+            url="http://www.example.com/this/list.html",
+        )
         req = self.request_class.from_response(response)
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers["Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(fs[b'test1'], [b'val1'])
-        self.assertEqual(fs[b'test2'], [b'val2'])
-        self.assertEqual(fs[b'button1'], [b'submit1'])
+        self.assertEqual(fs[b"test1"], [b"val1"])
+        self.assertEqual(fs[b"test2"], [b"val2"])
+        self.assertEqual(fs[b"button1"], [b"submit1"])
 
     def test_from_response_button_notype(self):
         response = _buildresponse(
@@ -1128,15 +1295,18 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="val2">
             <button name="button1" value="submit1">Submit</button>
             </form>""",
-            url="http://www.example.com/this/list.html")
+            url="http://www.example.com/this/list.html",
+        )
         req = self.request_class.from_response(response)
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers["Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(fs[b'test1'], [b'val1'])
-        self.assertEqual(fs[b'test2'], [b'val2'])
-        self.assertEqual(fs[b'button1'], [b'submit1'])
+        self.assertEqual(fs[b"test1"], [b"val1"])
+        self.assertEqual(fs[b"test2"], [b"val2"])
+        self.assertEqual(fs[b"button1"], [b"submit1"])
 
     def test_from_response_submit_novalue(self):
         response = _buildresponse(
@@ -1145,15 +1315,18 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="val2">
             <input type="submit" name="button1">Submit</button>
             </form>""",
-            url="http://www.example.com/this/list.html")
+            url="http://www.example.com/this/list.html",
+        )
         req = self.request_class.from_response(response)
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers["Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(fs[b'test1'], [b'val1'])
-        self.assertEqual(fs[b'test2'], [b'val2'])
-        self.assertEqual(fs[b'button1'], [b''])
+        self.assertEqual(fs[b"test1"], [b"val1"])
+        self.assertEqual(fs[b"test2"], [b"val2"])
+        self.assertEqual(fs[b"button1"], [b""])
 
     def test_from_response_button_novalue(self):
         response = _buildresponse(
@@ -1162,15 +1335,18 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="val2">
             <button type="submit" name="button1">Submit</button>
             </form>""",
-            url="http://www.example.com/this/list.html")
+            url="http://www.example.com/this/list.html",
+        )
         req = self.request_class.from_response(response)
-        self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.method, "POST")
+        self.assertEqual(
+            req.headers["Content-type"], b"application/x-www-form-urlencoded"
+        )
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(fs[b'test1'], [b'val1'])
-        self.assertEqual(fs[b'test2'], [b'val2'])
-        self.assertEqual(fs[b'button1'], [b''])
+        self.assertEqual(fs[b"test1"], [b"val1"])
+        self.assertEqual(fs[b"test2"], [b"val2"])
+        self.assertEqual(fs[b"button1"], [b""])
 
     def test_html_base_form_action(self):
         response = _buildresponse(
@@ -1185,15 +1361,15 @@ class FormRequestTest(RequestTest):
                 </body>
             </html>
             """,
-            url='http://a.com/'
+            url="http://a.com/",
         )
         req = self.request_class.from_response(response)
-        self.assertEqual(req.url, 'http://b.com/test_form')
+        self.assertEqual(req.url, "http://b.com/test_form")
 
     def test_spaces_in_action(self):
         resp = _buildresponse('<body><form action=" path\n"></form></body>')
         req = self.request_class.from_response(resp)
-        self.assertEqual(req.url, 'http://example.com/path')
+        self.assertEqual(req.url, "http://example.com/path")
 
     def test_from_response_css(self):
         response = _buildresponse(
@@ -1204,44 +1380,53 @@ class FormRequestTest(RequestTest):
             <form action="post2.php" method="POST">
             <input type="hidden" name="three" value="3">
             <input type="hidden" name="four" value="4">
-            </form>""")
-        r1 = self.request_class.from_response(response, formcss="form[action='post.php']")
+            </form>"""
+        )
+        r1 = self.request_class.from_response(
+            response, formcss="form[action='post.php']"
+        )
         fs = _qs(r1)
-        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b"one"], [b"1"])
 
         r1 = self.request_class.from_response(response, formcss="input[name='four']")
         fs = _qs(r1)
-        self.assertEqual(fs[b'three'], [b'3'])
+        self.assertEqual(fs[b"three"], [b"3"])
 
-        self.assertRaises(ValueError, self.request_class.from_response,
-                          response, formcss="input[name='abc']")
+        self.assertRaises(
+            ValueError,
+            self.request_class.from_response,
+            response,
+            formcss="input[name='abc']",
+        )
 
     def test_from_response_valid_form_methods(self):
-        form_methods = [[method, method] for method in self.request_class.valid_form_methods]
-        form_methods.append(['UNKNOWN', 'GET'])
+        form_methods = [
+            [method, method] for method in self.request_class.valid_form_methods
+        ]
+        form_methods.append(["UNKNOWN", "GET"])
 
         for method, expected in form_methods:
             response = _buildresponse(
                 f'<form action="post.php" method="{method}">'
                 '<input type="hidden" name="one" value="1">'
-                '</form>'
+                "</form>"
             )
             r = self.request_class.from_response(response)
             self.assertEqual(r.method, expected)
 
 
 def _buildresponse(body, **kwargs):
-    kwargs.setdefault('body', body)
-    kwargs.setdefault('url', 'http://example.com')
-    kwargs.setdefault('encoding', 'utf-8')
+    kwargs.setdefault("body", body)
+    kwargs.setdefault("url", "http://example.com")
+    kwargs.setdefault("encoding", "utf-8")
     return HtmlResponse(**kwargs)
 
 
-def _qs(req, encoding='utf-8', to_unicode=False):
-    if req.method == 'POST':
+def _qs(req, encoding="utf-8", to_unicode=False):
+    if req.method == "POST":
         qs = req.body
     else:
-        qs = req.url.partition('?')[2]
+        qs = req.url.partition("?")[2]
     uqs = unquote_to_bytes(qs)
     if to_unicode:
         uqs = uqs.decode(encoding)
@@ -1251,38 +1436,41 @@ def _qs(req, encoding='utf-8', to_unicode=False):
 class XmlRpcRequestTest(RequestTest):
 
     request_class = XmlRpcRequest
-    default_method = 'POST'
-    default_headers = {b'Content-Type': [b'text/xml']}
+    default_method = "POST"
+    default_headers = {b"Content-Type": [b"text/xml"]}
 
     def _test_request(self, **kwargs):
-        r = self.request_class('http://scrapytest.org/rpc2', **kwargs)
-        self.assertEqual(r.headers[b'Content-Type'], b'text/xml')
-        self.assertEqual(r.body,
-                         to_bytes(xmlrpc.client.dumps(**kwargs),
-                                  encoding=kwargs.get('encoding', 'utf-8')))
-        self.assertEqual(r.method, 'POST')
-        self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8'))
+        r = self.request_class("http://scrapytest.org/rpc2", **kwargs)
+        self.assertEqual(r.headers[b"Content-Type"], b"text/xml")
+        self.assertEqual(
+            r.body,
+            to_bytes(
+                xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8")
+            ),
+        )
+        self.assertEqual(r.method, "POST")
+        self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8"))
         self.assertTrue(r.dont_filter, True)
 
     def test_xmlrpc_dumps(self):
-        self._test_request(params=('value',))
-        self._test_request(params=('username', 'password'), methodname='login')
-        self._test_request(params=('response', ), methodresponse='login')
-        self._test_request(params=('pas£',), encoding='utf-8')
+        self._test_request(params=("value",))
+        self._test_request(params=("username", "password"), methodname="login")
+        self._test_request(params=("response",), methodresponse="login")
+        self._test_request(params=("pas£",), encoding="utf-8")
         self._test_request(params=(None,), allow_none=1)
         self.assertRaises(TypeError, self._test_request)
         self.assertRaises(TypeError, self._test_request, params=(None,))
 
     def test_latin1(self):
-        self._test_request(params=('pas£',), encoding='latin1')
+        self._test_request(params=("pas£",), encoding="latin1")
 
 
 class JsonRequestTest(RequestTest):
     request_class = JsonRequest
-    default_method = 'GET'
+    default_method = "GET"
     default_headers = {
-        b'Content-Type': [b'application/json'],
-        b'Accept': [b'application/json, text/javascript, */*; q=0.01'],
+        b"Content-Type": [b"application/json"],
+        b"Accept": [b"application/json, text/javascript, */*; q=0.01"],
     }
 
     def setUp(self):
@@ -1291,14 +1479,14 @@ class JsonRequestTest(RequestTest):
 
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
-        self.assertEqual(r1.body, b'')
+        self.assertEqual(r1.body, b"")
 
-        body = b'body'
+        body = b"body"
         r2 = self.request_class(url="http://www.example.com/", body=body)
         self.assertEqual(r2.body, body)
 
         data = {
-            'name': 'value',
+            "name": "value",
         }
         r3 = self.request_class(url="http://www.example.com/", data=data)
         self.assertEqual(r3.body, to_bytes(json.dumps(data)))
@@ -1310,134 +1498,138 @@ class JsonRequestTest(RequestTest):
     def test_data_method(self):
         # data is not passed
         r1 = self.request_class(url="http://www.example.com/")
-        self.assertEqual(r1.method, 'GET')
+        self.assertEqual(r1.method, "GET")
 
-        body = b'body'
+        body = b"body"
         r2 = self.request_class(url="http://www.example.com/", body=body)
-        self.assertEqual(r2.method, 'GET')
+        self.assertEqual(r2.method, "GET")
 
         data = {
-            'name': 'value',
+            "name": "value",
         }
         r3 = self.request_class(url="http://www.example.com/", data=data)
-        self.assertEqual(r3.method, 'POST')
+        self.assertEqual(r3.method, "POST")
 
         # method passed explicitly
-        r4 = self.request_class(url="http://www.example.com/", data=data, method='GET')
-        self.assertEqual(r4.method, 'GET')
+        r4 = self.request_class(url="http://www.example.com/", data=data, method="GET")
+        self.assertEqual(r4.method, "GET")
 
         r5 = self.request_class(url="http://www.example.com/", data=[])
-        self.assertEqual(r5.method, 'POST')
+        self.assertEqual(r5.method, "POST")
 
     def test_body_data(self):
-        """ passing both body and data should result a warning """
-        body = b'body'
+        """passing both body and data should result a warning"""
+        body = b"body"
         data = {
-            'name': 'value',
+            "name": "value",
         }
         with warnings.catch_warnings(record=True) as _warnings:
             r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
             self.assertEqual(r5.body, body)
-            self.assertEqual(r5.method, 'GET')
+            self.assertEqual(r5.method, "GET")
             self.assertEqual(len(_warnings), 1)
-            self.assertIn('data will be ignored', str(_warnings[0].message))
+            self.assertIn("data will be ignored", str(_warnings[0].message))
 
     def test_empty_body_data(self):
-        """ passing any body value and data should result a warning """
+        """passing any body value and data should result a warning"""
         data = {
-            'name': 'value',
+            "name": "value",
         }
         with warnings.catch_warnings(record=True) as _warnings:
-            r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)
-            self.assertEqual(r6.body, b'')
-            self.assertEqual(r6.method, 'GET')
+            r6 = self.request_class(url="http://www.example.com/", body=b"", data=data)
+            self.assertEqual(r6.body, b"")
+            self.assertEqual(r6.method, "GET")
             self.assertEqual(len(_warnings), 1)
-            self.assertIn('data will be ignored', str(_warnings[0].message))
+            self.assertIn("data will be ignored", str(_warnings[0].message))
 
     def test_body_none_data(self):
         data = {
-            'name': 'value',
+            "name": "value",
         }
         with warnings.catch_warnings(record=True) as _warnings:
             r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
             self.assertEqual(r7.body, to_bytes(json.dumps(data)))
-            self.assertEqual(r7.method, 'POST')
+            self.assertEqual(r7.method, "POST")
             self.assertEqual(len(_warnings), 0)
 
     def test_body_data_none(self):
         with warnings.catch_warnings(record=True) as _warnings:
             r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
-            self.assertEqual(r8.method, 'GET')
+            self.assertEqual(r8.method, "GET")
             self.assertEqual(len(_warnings), 0)
 
     def test_dumps_sort_keys(self):
-        """ Test that sort_keys=True is passed to json.dumps by default """
+        """Test that sort_keys=True is passed to json.dumps by default"""
         data = {
-            'name': 'value',
+            "name": "value",
         }
-        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+        with mock.patch("json.dumps", return_value=b"") as mock_dumps:
             self.request_class(url="http://www.example.com/", data=data)
             kwargs = mock_dumps.call_args[1]
-            self.assertEqual(kwargs['sort_keys'], True)
+            self.assertEqual(kwargs["sort_keys"], True)
 
     def test_dumps_kwargs(self):
-        """ Test that dumps_kwargs are passed to json.dumps """
+        """Test that dumps_kwargs are passed to json.dumps"""
         data = {
-            'name': 'value',
+            "name": "value",
         }
         dumps_kwargs = {
-            'ensure_ascii': True,
-            'allow_nan': True,
+            "ensure_ascii": True,
+            "allow_nan": True,
         }
-        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
-            self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs)
+        with mock.patch("json.dumps", return_value=b"") as mock_dumps:
+            self.request_class(
+                url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs
+            )
             kwargs = mock_dumps.call_args[1]
-            self.assertEqual(kwargs['ensure_ascii'], True)
-            self.assertEqual(kwargs['allow_nan'], True)
+            self.assertEqual(kwargs["ensure_ascii"], True)
+            self.assertEqual(kwargs["allow_nan"], True)
 
     def test_replace_data(self):
         data1 = {
-            'name1': 'value1',
+            "name1": "value1",
         }
         data2 = {
-            'name2': 'value2',
+            "name2": "value2",
         }
         r1 = self.request_class(url="http://www.example.com/", data=data1)
         r2 = r1.replace(data=data2)
         self.assertEqual(r2.body, to_bytes(json.dumps(data2)))
 
     def test_replace_sort_keys(self):
-        """ Test that replace provides sort_keys=True to json.dumps """
+        """Test that replace provides sort_keys=True to json.dumps"""
         data1 = {
-            'name1': 'value1',
+            "name1": "value1",
         }
         data2 = {
-            'name2': 'value2',
+            "name2": "value2",
         }
         r1 = self.request_class(url="http://www.example.com/", data=data1)
-        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+        with mock.patch("json.dumps", return_value=b"") as mock_dumps:
             r1.replace(data=data2)
             kwargs = mock_dumps.call_args[1]
-            self.assertEqual(kwargs['sort_keys'], True)
+            self.assertEqual(kwargs["sort_keys"], True)
 
     def test_replace_dumps_kwargs(self):
-        """ Test that dumps_kwargs are provided to json.dumps when replace is called """
+        """Test that dumps_kwargs are provided to json.dumps when replace is called"""
         data1 = {
-            'name1': 'value1',
+            "name1": "value1",
         }
         data2 = {
-            'name2': 'value2',
+            "name2": "value2",
         }
         dumps_kwargs = {
-            'ensure_ascii': True,
-            'allow_nan': True,
+            "ensure_ascii": True,
+            "allow_nan": True,
         }
-        r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs)
-        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+        r1 = self.request_class(
+            url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs
+        )
+        with mock.patch("json.dumps", return_value=b"") as mock_dumps:
             r1.replace(data=data2)
             kwargs = mock_dumps.call_args[1]
-            self.assertEqual(kwargs['ensure_ascii'], True)
-            self.assertEqual(kwargs['allow_nan'], True)
+            self.assertEqual(kwargs["ensure_ascii"], True)
+            self.assertEqual(kwargs["allow_nan"], True)
 
     def tearDown(self):
         warnings.resetwarnings()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index f51f3d988..c52fcce09 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -4,8 +4,14 @@ from unittest import mock
 
 from w3lib.encoding import resolve_encoding
 
-from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
-                         XmlResponse, Headers)
+from scrapy.http import (
+    Request,
+    Response,
+    TextResponse,
+    HtmlResponse,
+    XmlResponse,
+    Headers,
+)
 from scrapy.selector import Selector
 from scrapy.utils.python import to_unicode
 from scrapy.exceptions import NotSupported
@@ -20,15 +26,30 @@ class BaseResponseTest(unittest.TestCase):
     def test_init(self):
         # Response requires url in the constructor
         self.assertRaises(Exception, self.response_class)
-        self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
+        self.assertTrue(
+            isinstance(self.response_class("http://example.com/"), self.response_class)
+        )
         self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
-        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
-        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
+        self.assertTrue(
+            isinstance(
+                self.response_class("http://example.com/", body=b""),
+                self.response_class,
+            )
+        )
+        self.assertTrue(
+            isinstance(
+                self.response_class("http://example.com/", body=b"body"),
+                self.response_class,
+            )
+        )
         # test presence of all optional parameters
         self.assertTrue(
             isinstance(
-                self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class
+                self.response_class(
+                    "http://example.com/", body=b"", headers={}, status=200
+                ),
+                self.response_class,
             )
         )
 
@@ -49,15 +70,17 @@ class BaseResponseTest(unittest.TestCase):
 
         r = self.response_class("http://www.example.com", status=301)
         self.assertEqual(r.status, 301)
-        r = self.response_class("http://www.example.com", status='301')
+        r = self.response_class("http://www.example.com", status="301")
         self.assertEqual(r.status, 301)
-        self.assertRaises(ValueError, self.response_class, "http://example.com", status='lala200')
+        self.assertRaises(
+            ValueError, self.response_class, "http://example.com", status="lala200"
+        )
 
     def test_copy(self):
         """Test Response copy"""
 
         r1 = self.response_class("http://www.example.com", body=b"Some body")
-        r1.flags.append('cached')
+        r1.flags.append("cached")
         r2 = r1.copy()
 
         self.assertEqual(r1.status, r2.status)
@@ -68,29 +91,37 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(r1.flags, r2.flags)
 
         # make sure headers attribute is shallow copied
-        assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical"
+        assert (
+            r1.headers is not r2.headers
+        ), "headers must be a shallow copy, not identical"
         self.assertEqual(r1.headers, r2.headers)
 
     def test_copy_meta(self):
         req = Request("http://www.example.com")
-        req.meta['foo'] = 'bar'
-        r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
+        req.meta["foo"] = "bar"
+        r1 = self.response_class(
+            "http://www.example.com", body=b"Some body", request=req
+        )
         assert r1.meta is req.meta
 
     def test_copy_cb_kwargs(self):
         req = Request("http://www.example.com")
-        req.cb_kwargs['foo'] = 'bar'
-        r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
+        req.cb_kwargs["foo"] = "bar"
+        r1 = self.response_class(
+            "http://www.example.com", body=b"Some body", request=req
+        )
         assert r1.cb_kwargs is req.cb_kwargs
 
     def test_unavailable_meta(self):
         r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'):
+        with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"):
             r1.meta
 
     def test_unavailable_cb_kwargs(self):
         r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'):
+        with self.assertRaisesRegex(
+            AttributeError, r"Response\.cb_kwargs not available"
+        ):
             r1.cb_kwargs
 
     def test_copy_inherited_classes(self):
@@ -99,7 +130,7 @@ class BaseResponseTest(unittest.TestCase):
         class CustomResponse(self.response_class):
             pass
 
-        r1 = CustomResponse('http://www.example.com')
+        r1 = CustomResponse("http://www.example.com")
         r2 = r1.copy()
 
         assert isinstance(r2, CustomResponse)
@@ -109,16 +140,16 @@ class BaseResponseTest(unittest.TestCase):
         hdrs = Headers({"key": "value"})
         r1 = self.response_class("http://www.example.com")
         r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
-        assert r1.body == b''
+        assert r1.body == b""
         self.assertEqual(r1.url, r2.url)
         self.assertEqual((r1.status, r2.status), (200, 301))
-        self.assertEqual((r1.body, r2.body), (b'', b"New body"))
+        self.assertEqual((r1.body, r2.body), (b"", b"New body"))
         self.assertEqual((r1.headers, r2.headers), ({}, hdrs))
 
         # Empty attributes (which may fail if not compared properly)
-        r3 = self.response_class("http://www.example.com", flags=['cached'])
-        r4 = r3.replace(body=b'', flags=[])
-        self.assertEqual(r4.body, b'')
+        r3 = self.response_class("http://www.example.com", flags=["cached"])
+        r4 = r3.replace(body=b"", flags=[])
+        self.assertEqual(r4.body, b"")
         self.assertEqual(r4.flags, [])
 
     def _assert_response_values(self, response, encoding, body):
@@ -140,81 +171,84 @@ class BaseResponseTest(unittest.TestCase):
 
     def test_immutable_attributes(self):
         r = self.response_class("http://example.com")
-        self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
-        self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
+        self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
+        self.assertRaises(AttributeError, setattr, r, "body", "xxx")
 
     def test_urljoin(self):
         """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
-        joined = self.response_class('http://www.example.com').urljoin('/test')
-        absolute = 'http://www.example.com/test'
+        joined = self.response_class("http://www.example.com").urljoin("/test")
+        absolute = "http://www.example.com/test"
         self.assertEqual(joined, absolute)
 
     def test_shortcut_attributes(self):
-        r = self.response_class("http://example.com", body=b'hello')
+        r = self.response_class("http://example.com", body=b"hello")
         if self.response_class == Response:
             msg = "Response content isn't text"
-            self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text')
-            self.assertRaisesRegex(NotSupported, msg, r.css, 'body')
-            self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body')
+            self.assertRaisesRegex(AttributeError, msg, getattr, r, "text")
+            self.assertRaisesRegex(NotSupported, msg, r.css, "body")
+            self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body")
         else:
             r.text
-            r.css('body')
-            r.xpath('//body')
+            r.css("body")
+            r.xpath("//body")
 
     # Response.follow
 
     def test_follow_url_absolute(self):
-        self._assert_followed_url('http://foo.example.com',
-                                  'http://foo.example.com')
+        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
 
     def test_follow_url_relative(self):
-        self._assert_followed_url('foo',
-                                  'http://example.com/foo')
+        self._assert_followed_url("foo", "http://example.com/foo")
 
     def test_follow_link(self):
-        self._assert_followed_url(Link('http://example.com/foo'),
-                                  'http://example.com/foo')
+        self._assert_followed_url(
+            Link("http://example.com/foo"), "http://example.com/foo"
+        )
 
     def test_follow_None_url(self):
         r = self.response_class("http://example.com")
         self.assertRaises(ValueError, r.follow, None)
 
     def test_follow_whitespace_url(self):
-        self._assert_followed_url('foo ',
-                                  'http://example.com/foo%20')
+        self._assert_followed_url("foo ", "http://example.com/foo%20")
 
     def test_follow_whitespace_link(self):
-        self._assert_followed_url(Link('http://example.com/foo '),
-                                  'http://example.com/foo%20')
+        self._assert_followed_url(
+            Link("http://example.com/foo "), "http://example.com/foo%20"
+        )
 
     def test_follow_flags(self):
-        res = self.response_class('http://example.com/')
-        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
-        self.assertEqual(fol.flags, ['cached', 'allowed'])
+        res = self.response_class("http://example.com/")
+        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
+        self.assertEqual(fol.flags, ["cached", "allowed"])
 
     # Response.follow_all
 
     def test_follow_all_absolute(self):
-        url_list = ['http://example.org', 'http://www.example.org',
-                    'http://example.com', 'http://www.example.com']
+        url_list = [
+            "http://example.org",
+            "http://www.example.org",
+            "http://example.com",
+            "http://www.example.com",
+        ]
         self._assert_followed_all_urls(url_list, url_list)
 
     def test_follow_all_relative(self):
-        relative = ['foo', 'bar', 'foo/bar', 'bar/foo']
+        relative = ["foo", "bar", "foo/bar", "bar/foo"]
         absolute = [
-            'http://example.com/foo',
-            'http://example.com/bar',
-            'http://example.com/foo/bar',
-            'http://example.com/bar/foo',
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
         ]
         self._assert_followed_all_urls(relative, absolute)
 
     def test_follow_all_links(self):
         absolute = [
-            'http://example.com/foo',
-            'http://example.com/bar',
-            'http://example.com/foo/bar',
-            'http://example.com/bar/foo',
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
         ]
         links = map(Link, absolute)
         self._assert_followed_all_urls(links, absolute)
@@ -241,36 +275,36 @@ class BaseResponseTest(unittest.TestCase):
                 list(r.follow_all(urls=[None]))
 
     def test_follow_all_whitespace(self):
-        relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
+        relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
         absolute = [
-            'http://example.com/foo%20',
-            'http://example.com/bar%20',
-            'http://example.com/foo/bar%20',
-            'http://example.com/bar/foo%20',
+            "http://example.com/foo%20",
+            "http://example.com/bar%20",
+            "http://example.com/foo/bar%20",
+            "http://example.com/bar/foo%20",
         ]
         self._assert_followed_all_urls(relative, absolute)
 
     def test_follow_all_whitespace_links(self):
         absolute = [
-            'http://example.com/foo ',
-            'http://example.com/bar ',
-            'http://example.com/foo/bar ',
-            'http://example.com/bar/foo ',
+            "http://example.com/foo ",
+            "http://example.com/bar ",
+            "http://example.com/foo/bar ",
+            "http://example.com/bar/foo ",
         ]
         links = map(Link, absolute)
-        expected = [u.replace(' ', '%20') for u in absolute]
+        expected = [u.replace(" ", "%20") for u in absolute]
         self._assert_followed_all_urls(links, expected)
 
     def test_follow_all_flags(self):
-        re = self.response_class('http://www.example.com/')
+        re = self.response_class("http://www.example.com/")
         urls = [
-            'http://www.example.com/',
-            'http://www.example.com/2',
-            'http://www.example.com/foo',
+            "http://www.example.com/",
+            "http://www.example.com/2",
+            "http://www.example.com/foo",
         ]
-        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        fol = re.follow_all(urls, flags=["cached", "allowed"])
         for req in fol:
-            self.assertEqual(req.flags, ['cached', 'allowed'])
+            self.assertEqual(req.flags, ["cached", "allowed"])
 
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
@@ -288,13 +322,13 @@ class BaseResponseTest(unittest.TestCase):
             yield req
 
     def _links_response(self):
-        body = get_testdata('link_extractor', 'linkextractor.html')
-        resp = self.response_class('http://example.com/index', body=body)
+        body = get_testdata("link_extractor", "linkextractor.html")
+        resp = self.response_class("http://example.com/index", body=body)
         return resp
 
     def _links_response_no_href(self):
-        body = get_testdata('link_extractor', 'linkextractor_no_href.html')
-        resp = self.response_class('http://example.com/index', body=body)
+        body = get_testdata("link_extractor", "linkextractor_no_href.html")
+        resp = self.response_class("http://example.com/index", body=body)
         return resp
 
 
@@ -304,7 +338,9 @@ class TextResponseTest(BaseResponseTest):
 
     def test_replace(self):
         super().test_replace()
-        r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
+        r1 = self.response_class(
+            "http://www.example.com", body="hello", encoding="cp852"
+        )
         r2 = r1.replace(url="http://www.example.com/other")
         r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
 
@@ -320,52 +356,89 @@ class TextResponseTest(BaseResponseTest):
         self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
 
         # make sure urls are converted to str
-        resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
         assert isinstance(resp.url, str)
 
-        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
-        self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
-        resp = self.response_class("http://www.example.com/price/\xa3",
-                                   headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class("http://www.example.com/price/\xa3",
-                                   headers={"Content-type": ["text/html; charset=iso-8859-1"]})
-        self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
+        resp = self.response_class(
+            url="http://www.example.com/price/\xa3", encoding="utf-8"
+        )
+        self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
+        resp = self.response_class(
+            url="http://www.example.com/price/\xa3", encoding="latin-1"
+        )
+        self.assertEqual(resp.url, "http://www.example.com/price/\xa3")
+        resp = self.response_class(
+            "http://www.example.com/price/\xa3",
+            headers={"Content-type": ["text/html; charset=utf-8"]},
+        )
+        self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
+        resp = self.response_class(
+            "http://www.example.com/price/\xa3",
+            headers={"Content-type": ["text/html; charset=iso-8859-1"]},
+        )
+        self.assertEqual(resp.url, "http://www.example.com/price/\xa3")
 
     def test_unicode_body(self):
-        unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
-                          '\u0442\u0435\u043a\u0441\u0442')
-        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
+        unicode_string = (
+            "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 "
+            "\u0442\u0435\u043a\u0441\u0442"
+        )
+        self.assertRaises(
+            TypeError,
+            self.response_class,
+            "http://www.example.com",
+            body="unicode body",
+        )
 
-        original_string = unicode_string.encode('cp1251')
-        r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
+        original_string = unicode_string.encode("cp1251")
+        r1 = self.response_class(
+            "http://www.example.com", body=original_string, encoding="cp1251"
+        )
 
         # check response.text
         self.assertTrue(isinstance(r1.text, str))
         self.assertEqual(r1.text, unicode_string)
 
     def test_encoding(self):
-        r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
-                                 headers={"Content-type": ["text/html; charset=utf-8"]})
-        r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
-        r3 = self.response_class("http://www.example.com", body=b"\xa3",
-                                 headers={"Content-type": ["text/html; charset=iso-8859-1"]})
+        r1 = self.response_class(
+            "http://www.example.com",
+            body=b"\xc2\xa3",
+            headers={"Content-type": ["text/html; charset=utf-8"]},
+        )
+        r2 = self.response_class(
+            "http://www.example.com", encoding="utf-8", body="\xa3"
+        )
+        r3 = self.response_class(
+            "http://www.example.com",
+            body=b"\xa3",
+            headers={"Content-type": ["text/html; charset=iso-8859-1"]},
+        )
         r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
-        r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
-                                 headers={"Content-type": ["text/html; charset=None"]})
-        r6 = self.response_class("http://www.example.com", body=b"\xa8D",
-                                 headers={"Content-type": ["text/html; charset=gb2312"]})
-        r7 = self.response_class("http://www.example.com", body=b"\xa8D",
-                                 headers={"Content-type": ["text/html; charset=gbk"]})
-        r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3",
-                                 headers={"Content-type": ["text/html; charset=cp1251"]})
+        r5 = self.response_class(
+            "http://www.example.com",
+            body=b"\xc2\xa3",
+            headers={"Content-type": ["text/html; charset=None"]},
+        )
+        r6 = self.response_class(
+            "http://www.example.com",
+            body=b"\xa8D",
+            headers={"Content-type": ["text/html; charset=gb2312"]},
+        )
+        r7 = self.response_class(
+            "http://www.example.com",
+            body=b"\xa8D",
+            headers={"Content-type": ["text/html; charset=gbk"]},
+        )
+        r8 = self.response_class(
+            "http://www.example.com",
+            body=codecs.BOM_UTF8 + b"\xc2\xa3",
+            headers={"Content-type": ["text/html; charset=cp1251"]},
+        )
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
-        self.assertEqual(r2._declared_encoding(), 'utf-8')
-        self._assert_response_encoding(r2, 'utf-8')
+        self.assertEqual(r2._declared_encoding(), "utf-8")
+        self._assert_response_encoding(r2, "utf-8")
         self.assertEqual(r3._headers_encoding(), "cp1252")
         self.assertEqual(r3._declared_encoding(), "cp1252")
         self.assertEqual(r4._headers_encoding(), None)
@@ -374,81 +447,96 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r8._declared_encoding(), "utf-8")
         self._assert_response_encoding(r5, "utf-8")
         self._assert_response_encoding(r8, "utf-8")
-        assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
-        self._assert_response_values(r1, 'utf-8', "\xa3")
-        self._assert_response_values(r2, 'utf-8', "\xa3")
-        self._assert_response_values(r3, 'iso-8859-1', "\xa3")
-        self._assert_response_values(r6, 'gb18030', "\u2015")
-        self._assert_response_values(r7, 'gb18030', "\u2015")
+        assert (
+            r4._body_inferred_encoding() is not None
+            and r4._body_inferred_encoding() != "ascii"
+        )
+        self._assert_response_values(r1, "utf-8", "\xa3")
+        self._assert_response_values(r2, "utf-8", "\xa3")
+        self._assert_response_values(r3, "iso-8859-1", "\xa3")
+        self._assert_response_values(r6, "gb18030", "\u2015")
+        self._assert_response_values(r7, "gb18030", "\u2015")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
-        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
+        self.assertRaises(
+            TypeError, self.response_class, "http://www.example.com", body="\xa3"
+        )
 
     def test_declared_encoding_invalid(self):
         """Check that unknown declared encodings are ignored"""
-        r = self.response_class("http://www.example.com",
-                                headers={"Content-type": ["text/html; charset=UNKNOWN"]},
-                                body=b"\xc2\xa3")
+        r = self.response_class(
+            "http://www.example.com",
+            headers={"Content-type": ["text/html; charset=UNKNOWN"]},
+            body=b"\xc2\xa3",
+        )
         self.assertEqual(r._declared_encoding(), None)
-        self._assert_response_values(r, 'utf-8', "\xa3")
+        self._assert_response_values(r, "utf-8", "\xa3")
 
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
-        r = self.response_class("http://www.example.com",
-                                body=b'\xff\xfeh\x00i\x00',
-                                encoding='utf-16')
-        self._assert_response_values(r, 'utf-16', "hi")
+        r = self.response_class(
+            "http://www.example.com", body=b"\xff\xfeh\x00i\x00", encoding="utf-16"
+        )
+        self._assert_response_values(r, "utf-16", "hi")
 
     def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
-        r6 = self.response_class("http://www.example.com",
-                                 headers={"Content-type": ["text/html; charset=utf-8"]},
-                                 body=b"\xef\xbb\xbfWORD\xe3\xab")
-        self.assertEqual(r6.encoding, 'utf-8')
-        self.assertIn(r6.text, {
-            'WORD\ufffd\ufffd',  # w3lib < 1.19.0
-            'WORD\ufffd',        # w3lib >= 1.19.0
-        })
+        r6 = self.response_class(
+            "http://www.example.com",
+            headers={"Content-type": ["text/html; charset=utf-8"]},
+            body=b"\xef\xbb\xbfWORD\xe3\xab",
+        )
+        self.assertEqual(r6.encoding, "utf-8")
+        self.assertIn(
+            r6.text,
+            {
+                "WORD\ufffd\ufffd",  # w3lib < 1.19.0
+                "WORD\ufffd",  # w3lib >= 1.19.0
+            },
+        )
 
     def test_bom_is_removed_from_body(self):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
         # response.text in indistint order doesn't affect final
         # values for encoding and decoded body.
-        url = 'http://example.com'
+        url = "http://example.com"
         body = b"\xef\xbb\xbfWORD"
         headers = {"Content-type": ["text/html; charset=utf-8"]}
 
         # Test response without content-type and BOM encoding
         response = self.response_class(url, body=body)
-        self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, 'WORD')
+        self.assertEqual(response.encoding, "utf-8")
+        self.assertEqual(response.text, "WORD")
         response = self.response_class(url, body=body)
-        self.assertEqual(response.text, 'WORD')
-        self.assertEqual(response.encoding, 'utf-8')
+        self.assertEqual(response.text, "WORD")
+        self.assertEqual(response.encoding, "utf-8")
 
         # Body caching sideeffect isn't triggered when encoding is declared in
         # content-type header but BOM still need to be removed from decoded
         # body
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, 'WORD')
+        self.assertEqual(response.encoding, "utf-8")
+        self.assertEqual(response.text, "WORD")
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.text, 'WORD')
-        self.assertEqual(response.encoding, 'utf-8')
+        self.assertEqual(response.text, "WORD")
+        self.assertEqual(response.encoding, "utf-8")
 
     def test_replace_wrong_encoding(self):
         """Test invalid chars are replaced properly"""
-        r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
+        r = self.response_class(
+            "http://www.example.com", encoding="utf-8", body=b"PREFIX\xe3\xabSUFFIX"
+        )
         # XXX: Policy for replacing invalid chars may suffer minor variations
         # but it should always contain the unicode replacement char ('\ufffd')
-        assert '\ufffd' in r.text, repr(r.text)
-        assert 'PREFIX' in r.text, repr(r.text)
-        assert 'SUFFIX' in r.text, repr(r.text)
+        assert "\ufffd" in r.text, repr(r.text)
+        assert "PREFIX" in r.text, repr(r.text)
+        assert "SUFFIX" in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
-        r = self.response_class("http://example.com", encoding='utf-8',
-                                body=b'\xf0<span>value</span>')
-        assert '<span>value</span>' in r.text, repr(r.text)
+        r = self.response_class(
+            "http://example.com", encoding="utf-8", body=b"\xf0<span>value</span>"
+        )
+        assert "<span>value</span>" in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
         # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
@@ -459,22 +547,15 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertIsInstance(response.selector, Selector)
-        self.assertEqual(response.selector.type, 'html')
+        self.assertEqual(response.selector.type, "html")
         self.assertIs(response.selector, response.selector)  # property is cached
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//title/text()").getall(),
-            ['Some page']
-        )
-        self.assertEqual(
-            response.selector.css("title::text").getall(),
-            ['Some page']
-        )
-        self.assertEqual(
-            response.selector.re("Some (.*)</title>"),
-            ['page']
+            response.selector.xpath("//title/text()").getall(), ["Some page"]
         )
+        self.assertEqual(response.selector.css("title::text").getall(), ["Some page"])
+        self.assertEqual(response.selector.re("Some (.*)</title>"), ["page"])
 
     def test_selector_shortcuts(self):
         body = b"<html><head><title>Some page</title><body></body></html>"
@@ -490,151 +571,163 @@ class TextResponseTest(BaseResponseTest):
         )
 
     def test_selector_shortcuts_kwargs(self):
-        body = b"<html><head><title>Some page</title><body><p class=\"content\">A nice paragraph.</p></body></html>"
+        body = b'<html><head><title>Some page</title><body><p class="content">A nice paragraph.</p></body></html>'
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
-            response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
+            response.xpath(
+                "normalize-space(//p[@class=$pclass])", pclass="content"
+            ).getall(),
+            response.xpath('normalize-space(//p[@class="content"])').getall(),
         )
         self.assertEqual(
             response.xpath(
                 "//title[count(following::p[@class=$pclass])=$pcount]/text()",
-                pclass="content", pcount=1,
+                pclass="content",
+                pcount=1,
+            ).getall(),
+            response.xpath(
+                '//title[count(following::p[@class="content"])=1]/text()'
             ).getall(),
-            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
         )
 
     def test_urljoin_with_base_url(self):
         """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
         body = b'<html><body><base href="https://example.net"></body></html>'
-        joined = self.response_class('http://www.example.com', body=body).urljoin('/test')
-        absolute = 'https://example.net/test'
+        joined = self.response_class("http://www.example.com", body=body).urljoin(
+            "/test"
+        )
+        absolute = "https://example.net/test"
         self.assertEqual(joined, absolute)
 
         body = b'<html><body><base href="/elsewhere"></body></html>'
-        joined = self.response_class('http://www.example.com', body=body).urljoin('test')
-        absolute = 'http://www.example.com/test'
+        joined = self.response_class("http://www.example.com", body=body).urljoin(
+            "test"
+        )
+        absolute = "http://www.example.com/test"
         self.assertEqual(joined, absolute)
 
         body = b'<html><body><base href="/elsewhere/"></body></html>'
-        joined = self.response_class('http://www.example.com', body=body).urljoin('test')
-        absolute = 'http://www.example.com/elsewhere/test'
+        joined = self.response_class("http://www.example.com", body=body).urljoin(
+            "test"
+        )
+        absolute = "http://www.example.com/elsewhere/test"
         self.assertEqual(joined, absolute)
 
     def test_follow_selector(self):
         resp = self._links_response()
         urls = [
-            'http://example.com/sample2.html',
-            'http://example.com/sample3.html',
-            'http://example.com/sample3.html',
-            'http://example.com/sample3.html#foo',
-            'http://www.google.com/something',
-            'http://example.com/innertag.html'
+            "http://example.com/sample2.html",
+            "http://example.com/sample3.html",
+            "http://example.com/sample3.html",
+            "http://example.com/sample3.html#foo",
+            "http://www.google.com/something",
+            "http://example.com/innertag.html",
         ]
 
         # select <a> elements
-        for sellist in [resp.css('a'), resp.xpath('//a')]:
+        for sellist in [resp.css("a"), resp.xpath("//a")]:
             for sel, url in zip(sellist, urls):
                 self._assert_followed_url(sel, url, response=resp)
 
         # select <link> elements
         self._assert_followed_url(
-            Selector(text='<link href="foo"></link>').css('link')[0],
-            'http://example.com/foo',
-            response=resp
+            Selector(text='<link href="foo"></link>').css("link")[0],
+            "http://example.com/foo",
+            response=resp,
         )
 
         # href attributes should work
-        for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
+        for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
             for sel, url in zip(sellist, urls):
                 self._assert_followed_url(sel, url, response=resp)
 
         # non-a elements are not supported
-        self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
+        self.assertRaises(ValueError, resp.follow, resp.css("div")[0])
 
     def test_follow_selector_list(self):
         resp = self._links_response()
-        self.assertRaisesRegex(ValueError, 'SelectorList',
-                               resp.follow, resp.css('a'))
+        self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a"))
 
     def test_follow_selector_invalid(self):
         resp = self._links_response()
-        self.assertRaisesRegex(ValueError, 'Unsupported',
-                               resp.follow, resp.xpath('count(//div)')[0])
+        self.assertRaisesRegex(
+            ValueError, "Unsupported", resp.follow, resp.xpath("count(//div)")[0]
+        )
 
     def test_follow_selector_attribute(self):
         resp = self._links_response()
-        for src in resp.css('img::attr(src)'):
-            self._assert_followed_url(src, 'http://example.com/sample2.jpg')
+        for src in resp.css("img::attr(src)"):
+            self._assert_followed_url(src, "http://example.com/sample2.jpg")
 
     def test_follow_selector_no_href(self):
         resp = self.response_class(
-            url='http://example.com',
-            body=b'<html><body><a name=123>click me</a></body></html>',
+            url="http://example.com",
+            body=b"<html><body><a name=123>click me</a></body></html>",
         )
-        self.assertRaisesRegex(ValueError, 'no href',
-                               resp.follow, resp.css('a')[0])
+        self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0])
 
     def test_follow_whitespace_selector(self):
         resp = self.response_class(
-            'http://example.com',
-            body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
+            "http://example.com",
+            body=b"""<html><body><a href=" foo\n">click me</a></body></html>""",
         )
         self._assert_followed_url(
-            resp.css('a')[0],
-            'http://example.com/foo',
-            response=resp)
+            resp.css("a")[0], "http://example.com/foo", response=resp
+        )
         self._assert_followed_url(
-            resp.css('a::attr(href)')[0],
-            'http://example.com/foo',
-            response=resp)
+            resp.css("a::attr(href)")[0], "http://example.com/foo", response=resp
+        )
 
     def test_follow_encoding(self):
         resp1 = self.response_class(
-            'http://example.com',
-            encoding='utf8',
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+            "http://example.com",
+            encoding="utf8",
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
+                "utf8"
+            ),
         )
         req = self._assert_followed_url(
-            resp1.css('a')[0],
-            'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
+            resp1.css("a")[0],
+            "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
             response=resp1,
         )
-        self.assertEqual(req.encoding, 'utf8')
+        self.assertEqual(req.encoding, "utf8")
 
         resp2 = self.response_class(
-            'http://example.com',
-            encoding='cp1251',
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+            "http://example.com",
+            encoding="cp1251",
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
+                "cp1251"
+            ),
         )
         req = self._assert_followed_url(
-            resp2.css('a')[0],
-            'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
+            resp2.css("a")[0],
+            "http://example.com/foo?%EF%F0%E8%E2%E5%F2",
             response=resp2,
         )
-        self.assertEqual(req.encoding, 'cp1251')
+        self.assertEqual(req.encoding, "cp1251")
 
     def test_follow_flags(self):
-        res = self.response_class('http://example.com/')
-        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
-        self.assertEqual(fol.flags, ['cached', 'allowed'])
+        res = self.response_class("http://example.com/")
+        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
+        self.assertEqual(fol.flags, ["cached", "allowed"])
 
     def test_follow_all_flags(self):
-        re = self.response_class('http://www.example.com/')
+        re = self.response_class("http://www.example.com/")
         urls = [
-            'http://www.example.com/',
-            'http://www.example.com/2',
-            'http://www.example.com/foo',
+            "http://www.example.com/",
+            "http://www.example.com/2",
+            "http://www.example.com/foo",
         ]
-        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        fol = re.follow_all(urls, flags=["cached", "allowed"])
         for req in fol:
-            self.assertEqual(req.flags, ['cached', 'allowed'])
+            self.assertEqual(req.flags, ["cached", "allowed"])
 
     def test_follow_all_css(self):
         expected = [
-            'http://example.com/sample3.html',
-            'http://example.com/innertag.html',
+            "http://example.com/sample3.html",
+            "http://example.com/innertag.html",
         ]
         response = self._links_response()
         extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
@@ -642,20 +735,20 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_all_css_skip_invalid(self):
         expected = [
-            'http://example.com/page/1/',
-            'http://example.com/page/3/',
-            'http://example.com/page/4/',
+            "http://example.com/page/1/",
+            "http://example.com/page/3/",
+            "http://example.com/page/4/",
         ]
         response = self._links_response_no_href()
-        extracted1 = [r.url for r in response.follow_all(css='.pagination a')]
+        extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
         self.assertEqual(expected, extracted1)
-        extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))]
+        extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
         self.assertEqual(expected, extracted2)
 
     def test_follow_all_xpath(self):
         expected = [
-            'http://example.com/sample3.html',
-            'http://example.com/innertag.html',
+            "http://example.com/sample3.html",
+            "http://example.com/innertag.html",
         ]
         response = self._links_response()
         extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
@@ -663,25 +756,33 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_all_xpath_skip_invalid(self):
         expected = [
-            'http://example.com/page/1/',
-            'http://example.com/page/3/',
-            'http://example.com/page/4/',
+            "http://example.com/page/1/",
+            "http://example.com/page/3/",
+            "http://example.com/page/4/",
         ]
         response = self._links_response_no_href()
-        extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
+        extracted1 = [
+            r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
+        ]
         self.assertEqual(expected, extracted1)
-        extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))]
+        extracted2 = [
+            r.url
+            for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
+        ]
         self.assertEqual(expected, extracted2)
 
     def test_follow_all_too_many_arguments(self):
         response = self._links_response()
         with self.assertRaises(ValueError):
-            response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
+            response.follow_all(
+                css='a[href*="example.com"]',
+                xpath='//a[contains(@href, "example.com")]',
+            )
 
     def test_json_response(self):
         json_body = b"""{"ip": "109.187.217.200"}"""
         json_response = self.response_class("http://www.example.com", body=json_body)
-        self.assertEqual(json_response.json(), {'ip': '109.187.217.200'})
+        self.assertEqual(json_response.json(), {"ip": "109.187.217.200"})
 
         text_body = b"""<html><body>text</body></html>"""
         text_response = self.response_class("http://www.example.com", body=text_body)
@@ -691,9 +792,11 @@ class TextResponseTest(BaseResponseTest):
     def test_cache_json_response(self):
         json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
         for json_body in json_valid_bodies:
-            json_response = self.response_class("http://www.example.com", body=json_body)
+            json_response = self.response_class(
+                "http://www.example.com", body=json_body
+            )
 
-            with mock.patch('json.loads') as mock_json:
+            with mock.patch("json.loads") as mock_json:
                 for _ in range(2):
                     json_response.json()
                 mock_json.assert_called_once_with(json_body.decode())
@@ -710,33 +813,36 @@ class HtmlResponseTest(TextResponseTest):
         </head><body>Price: \xa3100</body></html>'
         """
         r1 = self.response_class("http://www.example.com", body=body)
-        self._assert_response_values(r1, 'iso-8859-1', body)
+        self._assert_response_values(r1, "iso-8859-1", body)
 
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?>
         <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
         Price: \xa3100
         """
         r2 = self.response_class("http://www.example.com", body=body)
-        self._assert_response_values(r2, 'iso-8859-1', body)
+        self._assert_response_values(r2, "iso-8859-1", body)
 
         # for conflicting declarations headers must take precedence
         body = b"""<html><head><title>Some page</title>
         <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
-        r3 = self.response_class("http://www.example.com", body=body,
-                                 headers={"Content-type": ["text/html; charset=iso-8859-1"]})
-        self._assert_response_values(r3, 'iso-8859-1', body)
+        r3 = self.response_class(
+            "http://www.example.com",
+            body=body,
+            headers={"Content-type": ["text/html; charset=iso-8859-1"]},
+        )
+        self._assert_response_values(r3, "iso-8859-1", body)
 
         # make sure replace() preserves the encoding of the original response
         body = b"New body \xa3"
         r4 = r3.replace(body=body)
-        self._assert_response_values(r4, 'iso-8859-1', body)
+        self._assert_response_values(r4, "iso-8859-1", body)
 
     def test_html5_meta_charset(self):
         body = b"""<html><head><meta charset="gb2312" /><title>Some page</title><body>bla bla</body>"""
         r1 = self.response_class("http://www.example.com", body=body)
-        self._assert_response_values(r1, 'gb2312', body)
+        self._assert_response_values(r1, "gb2312", body)
 
 
 class XmlResponseTest(TextResponseTest):
@@ -750,14 +856,14 @@ class XmlResponseTest(TextResponseTest):
 
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
         r2 = self.response_class("http://www.example.com", body=body)
-        self._assert_response_values(r2, 'iso-8859-1', body)
+        self._assert_response_values(r2, "iso-8859-1", body)
 
         # make sure replace() preserves the explicit encoding passed in the __init__ method
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
-        r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8')
+        r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8")
         body2 = b"New body"
         r4 = r3.replace(body=body2)
-        self._assert_response_values(r4, 'utf-8', body2)
+        self._assert_response_values(r4, "utf-8", body2)
 
     def test_replace_encoding(self):
         # make sure replace() keeps the previous encoding unless overridden explicitly
@@ -765,24 +871,21 @@ class XmlResponseTest(TextResponseTest):
         body2 = b"""<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
         r5 = self.response_class("http://www.example.com", body=body)
         r6 = r5.replace(body=body2)
-        r7 = r5.replace(body=body2, encoding='utf-8')
-        self._assert_response_values(r5, 'iso-8859-1', body)
-        self._assert_response_values(r6, 'iso-8859-1', body2)
-        self._assert_response_values(r7, 'utf-8', body2)
+        r7 = r5.replace(body=body2, encoding="utf-8")
+        self._assert_response_values(r5, "iso-8859-1", body)
+        self._assert_response_values(r6, "iso-8859-1", body2)
+        self._assert_response_values(r7, "utf-8", body2)
 
     def test_selector(self):
         body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertIsInstance(response.selector, Selector)
-        self.assertEqual(response.selector.type, 'xml')
+        self.assertEqual(response.selector.type, "xml")
         self.assertIs(response.selector, response.selector)  # property is cached
         self.assertIs(response.selector.response, response)
 
-        self.assertEqual(
-            response.selector.xpath("//elem/text()").getall(),
-            ['value']
-        )
+        self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"])
 
     def test_selector_shortcuts(self):
         body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
@@ -794,20 +897,26 @@ class XmlResponseTest(TextResponseTest):
         )
 
     def test_selector_shortcuts_kwargs(self):
-        body = b'''<?xml version="1.0" encoding="utf-8"?>
+        body = b"""<?xml version="1.0" encoding="utf-8"?>
         <xml xmlns:somens="http://scrapy.org">
         <somens:elem>value</somens:elem>
-        </xml>'''
+        </xml>"""
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
-            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
+            response.xpath(
+                "//s:elem/text()", namespaces={"s": "http://scrapy.org"}
+            ).getall(),
+            response.selector.xpath(
+                "//s:elem/text()", namespaces={"s": "http://scrapy.org"}
+            ).getall(),
         )
 
-        response.selector.register_namespace('s2', 'http://scrapy.org')
+        response.selector.register_namespace("s2", "http://scrapy.org")
         self.assertEqual(
-            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
+            response.xpath(
+                "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"}
+            ).getall(),
             response.selector.xpath("//s2:elem/text()").getall(),
         )
 
@@ -827,7 +936,9 @@ class CustomResponseTest(TextResponseTest):
 
     def test_copy(self):
         super().test_copy()
-        r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+        r1 = self.response_class(
+            url="https://example.org", status=200, foo="foo", bar="bar", lost="lost"
+        )
         r2 = r1.copy()
         self.assertIsInstance(r2, self.response_class)
         self.assertEqual(r1.foo, r2.foo)
@@ -837,7 +948,9 @@ class CustomResponseTest(TextResponseTest):
 
     def test_replace(self):
         super().test_replace()
-        r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+        r1 = self.response_class(
+            url="https://example.org", status=200, foo="foo", bar="bar", lost="lost"
+        )
 
         r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
         self.assertIsInstance(r2, self.response_class)
@@ -868,4 +981,8 @@ class CustomResponseTest(TextResponseTest):
 
         with self.assertRaises(TypeError) as ctx:
             r1.replace(unknown="unknown")
-        self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'"))
+        self.assertTrue(
+            str(ctx.exception).endswith(
+                "__init__() got an unexpected keyword argument 'unknown'"
+            )
+        )
diff --git a/tests/test_item.py b/tests/test_item.py
index 25f2aea0a..ce2b4fd15 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -5,7 +5,6 @@ from scrapy.item import ABCMeta, Field, Item, ItemMeta
 
 
 class ItemTest(unittest.TestCase):
-
     def assertSortedEqual(self, first, second, msg=None):
         return self.assertEqual(sorted(first), sorted(second), msg)
 
@@ -14,35 +13,34 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = 'name'
-        self.assertEqual(i['name'], 'name')
+        i["name"] = "name"
+        self.assertEqual(i["name"], "name")
 
     def test_init(self):
         class TestItem(Item):
             name = Field()
 
         i = TestItem()
-        self.assertRaises(KeyError, i.__getitem__, 'name')
+        self.assertRaises(KeyError, i.__getitem__, "name")
 
-        i2 = TestItem(name='john doe')
-        self.assertEqual(i2['name'], 'john doe')
+        i2 = TestItem(name="john doe")
+        self.assertEqual(i2["name"], "john doe")
 
-        i3 = TestItem({'name': 'john doe'})
-        self.assertEqual(i3['name'], 'john doe')
+        i3 = TestItem({"name": "john doe"})
+        self.assertEqual(i3["name"], "john doe")
 
         i4 = TestItem(i3)
-        self.assertEqual(i4['name'], 'john doe')
+        self.assertEqual(i4["name"], "john doe")
 
-        self.assertRaises(KeyError, TestItem, {'name': 'john doe',
-                                               'other': 'foo'})
+        self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"})
 
     def test_invalid_field(self):
         class TestItem(Item):
             pass
 
         i = TestItem()
-        self.assertRaises(KeyError, i.__setitem__, 'field', 'text')
-        self.assertRaises(KeyError, i.__getitem__, 'field')
+        self.assertRaises(KeyError, i.__setitem__, "field", "text")
+        self.assertRaises(KeyError, i.__getitem__, "field")
 
     def test_repr(self):
         class TestItem(Item):
@@ -50,55 +48,54 @@ class ItemTest(unittest.TestCase):
             number = Field()
 
         i = TestItem()
-        i['name'] = 'John Doe'
-        i['number'] = 123
+        i["name"] = "John Doe"
+        i["number"] = 123
         itemrepr = repr(i)
 
-        self.assertEqual(itemrepr,
-                         "{'name': 'John Doe', 'number': 123}")
+        self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}")
 
         i2 = eval(itemrepr)
-        self.assertEqual(i2['name'], 'John Doe')
-        self.assertEqual(i2['number'], 123)
+        self.assertEqual(i2["name"], "John Doe")
+        self.assertEqual(i2["number"], 123)
 
     def test_private_attr(self):
         class TestItem(Item):
             name = Field()
 
         i = TestItem()
-        i._private = 'test'
-        self.assertEqual(i._private, 'test')
+        i._private = "test"
+        self.assertEqual(i._private, "test")
 
     def test_raise_getattr(self):
         class TestItem(Item):
             name = Field()
 
         i = TestItem()
-        self.assertRaises(AttributeError, getattr, i, 'name')
+        self.assertRaises(AttributeError, getattr, i, "name")
 
     def test_raise_setattr(self):
         class TestItem(Item):
             name = Field()
 
         i = TestItem()
-        self.assertRaises(AttributeError, setattr, i, 'name', 'john')
+        self.assertRaises(AttributeError, setattr, i, "name", "john")
 
     def test_custom_methods(self):
         class TestItem(Item):
             name = Field()
 
             def get_name(self):
-                return self['name']
+                return self["name"]
 
             def change_name(self, name):
-                self['name'] = name
+                self["name"] = name
 
         i = TestItem()
         self.assertRaises(KeyError, i.get_name)
-        i['name'] = 'lala'
-        self.assertEqual(i.get_name(), 'lala')
-        i.change_name('other')
-        self.assertEqual(i.get_name(), 'other')
+        i["name"] = "lala"
+        self.assertEqual(i.get_name(), "lala")
+        i.change_name("other")
+        self.assertEqual(i.get_name(), "other")
 
     def test_metaclass(self):
         class TestItem(Item):
@@ -107,22 +104,22 @@ class ItemTest(unittest.TestCase):
             values = Field()
 
         i = TestItem()
-        i['name'] = 'John'
-        self.assertEqual(list(i.keys()), ['name'])
-        self.assertEqual(list(i.values()), ['John'])
+        i["name"] = "John"
+        self.assertEqual(list(i.keys()), ["name"])
+        self.assertEqual(list(i.values()), ["John"])
 
-        i['keys'] = 'Keys'
-        i['values'] = 'Values'
-        self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
-        self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
+        i["keys"] = "Keys"
+        i["values"] = "Values"
+        self.assertSortedEqual(list(i.keys()), ["keys", "values", "name"])
+        self.assertSortedEqual(list(i.values()), ["Keys", "Values", "John"])
 
     def test_metaclass_with_fields_attribute(self):
         class TestItem(Item):
-            fields = {'new': Field(default='X')}
+            fields = {"new": Field(default="X")}
 
-        item = TestItem(new='New')
-        self.assertSortedEqual(list(item.keys()), ['new'])
-        self.assertSortedEqual(list(item.values()), ['New'])
+        item = TestItem(new="New")
+        self.assertSortedEqual(list(item.keys()), ["new"])
+        self.assertSortedEqual(list(item.values()), ["New"])
 
     def test_metaclass_inheritance(self):
         class ParentItem(Item):
@@ -134,151 +131,163 @@ class ItemTest(unittest.TestCase):
             keys = Field()
 
         i = TestItem()
-        i['keys'] = 3
-        self.assertEqual(list(i.keys()), ['keys'])
+        i["keys"] = 3
+        self.assertEqual(list(i.keys()), ["keys"])
         self.assertEqual(list(i.values()), [3])
 
     def test_metaclass_multiple_inheritance_simple(self):
         class A(Item):
-            fields = {'load': Field(default='A')}
-            save = Field(default='A')
+            fields = {"load": Field(default="A")}
+            save = Field(default="A")
 
         class B(A):
             pass
 
         class C(Item):
-            fields = {'load': Field(default='C')}
-            save = Field(default='C')
+            fields = {"load": Field(default="C")}
+            save = Field(default="C")
 
         class D(B, C):
             pass
 
-        item = D(save='X', load='Y')
-        self.assertEqual(item['save'], 'X')
-        self.assertEqual(item['load'], 'Y')
-        self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})
+        item = D(save="X", load="Y")
+        self.assertEqual(item["save"], "X")
+        self.assertEqual(item["load"], "Y")
+        self.assertEqual(D.fields, {"load": {"default": "A"}, "save": {"default": "A"}})
 
         # D class inverted
         class E(C, B):
             pass
 
-        self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E(load='X')['load'], 'X')
-        self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})
+        self.assertEqual(E(save="X")["save"], "X")
+        self.assertEqual(E(load="X")["load"], "X")
+        self.assertEqual(E.fields, {"load": {"default": "C"}, "save": {"default": "C"}})
 
     def test_metaclass_multiple_inheritance_diamond(self):
         class A(Item):
-            fields = {'update': Field(default='A')}
-            save = Field(default='A')
-            load = Field(default='A')
+            fields = {"update": Field(default="A")}
+            save = Field(default="A")
+            load = Field(default="A")
 
         class B(A):
             pass
 
         class C(A):
-            fields = {'update': Field(default='C')}
-            save = Field(default='C')
+            fields = {"update": Field(default="C")}
+            save = Field(default="C")
 
         class D(B, C):
-            fields = {'update': Field(default='D')}
-            load = Field(default='D')
+            fields = {"update": Field(default="D")}
+            load = Field(default="D")
 
-        self.assertEqual(D(save='X')['save'], 'X')
-        self.assertEqual(D(load='X')['load'], 'X')
+        self.assertEqual(D(save="X")["save"], "X")
+        self.assertEqual(D(load="X")["load"], "X")
         self.assertEqual(
             D.fields,
-            {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})
+            {
+                "save": {"default": "C"},
+                "load": {"default": "D"},
+                "update": {"default": "D"},
+            },
+        )
 
         # D class inverted
         class E(C, B):
-            load = Field(default='E')
+            load = Field(default="E")
 
-        self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E(load='X')['load'], 'X')
+        self.assertEqual(E(save="X")["save"], "X")
+        self.assertEqual(E(load="X")["load"], "X")
         self.assertEqual(
             E.fields,
-            {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})
+            {
+                "save": {"default": "C"},
+                "load": {"default": "E"},
+                "update": {"default": "C"},
+            },
+        )
 
     def test_metaclass_multiple_inheritance_without_metaclass(self):
         class A(Item):
-            fields = {'load': Field(default='A')}
-            save = Field(default='A')
+            fields = {"load": Field(default="A")}
+            save = Field(default="A")
 
         class B(A):
             pass
 
         class C:
-            fields = {'load': Field(default='C')}
-            not_allowed = Field(default='not_allowed')
-            save = Field(default='C')
+            fields = {"load": Field(default="C")}
+            not_allowed = Field(default="not_allowed")
+            save = Field(default="C")
 
         class D(B, C):
             pass
 
-        self.assertRaises(KeyError, D, not_allowed='value')
-        self.assertEqual(D(save='X')['save'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
+        self.assertRaises(KeyError, D, not_allowed="value")
+        self.assertEqual(D(save="X")["save"], "X")
+        self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}})
 
         # D class inverted
         class E(C, B):
             pass
 
-        self.assertRaises(KeyError, E, not_allowed='value')
-        self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
+        self.assertRaises(KeyError, E, not_allowed="value")
+        self.assertEqual(E(save="X")["save"], "X")
+        self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}})
 
     def test_to_dict(self):
         class TestItem(Item):
             name = Field()
 
         i = TestItem()
-        i['name'] = 'John'
-        self.assertEqual(dict(i), {'name': 'John'})
+        i["name"] = "John"
+        self.assertEqual(dict(i), {"name": "John"})
 
     def test_copy(self):
         class TestItem(Item):
             name = Field()
-        item = TestItem({'name': 'lower'})
+
+        item = TestItem({"name": "lower"})
         copied_item = item.copy()
         self.assertNotEqual(id(item), id(copied_item))
-        copied_item['name'] = copied_item['name'].upper()
-        self.assertNotEqual(item['name'], copied_item['name'])
+        copied_item["name"] = copied_item["name"].upper()
+        self.assertNotEqual(item["name"], copied_item["name"])
 
     def test_deepcopy(self):
         class TestItem(Item):
             tags = Field()
-        item = TestItem({'tags': ['tag1']})
+
+        item = TestItem({"tags": ["tag1"]})
         copied_item = item.deepcopy()
-        item['tags'].append('tag2')
-        assert item['tags'] != copied_item['tags']
+        item["tags"].append("tag2")
+        assert item["tags"] != copied_item["tags"]
 
 
 class ItemMetaTest(unittest.TestCase):
-
     def test_new_method_propagates_classcell(self):
         new_mock = mock.Mock(side_effect=ABCMeta.__new__)
         base = ItemMeta.__bases__[0]
 
-        with mock.patch.object(base, '__new__', new_mock):
+        with mock.patch.object(base, "__new__", new_mock):
 
             class MyItem(Item):
                 def f(self):
                     # For rationale of this see:
                     # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
-                    return __class__  # noqa  https://github.com/scrapy/scrapy/issues/2836
+                    return (
+                        __class__  # noqa  https://github.com/scrapy/scrapy/issues/2836
+                    )
 
             MyItem()
 
         (first_call, second_call) = new_mock.call_args_list[-2:]
 
         mcs, class_name, bases, attrs = first_call[0]
-        assert '__classcell__' not in attrs
+        assert "__classcell__" not in attrs
         mcs, class_name, bases, attrs = second_call[0]
-        assert '__classcell__' in attrs
+        assert "__classcell__" in attrs
 
 
 class ItemMetaClassCellRegression(unittest.TestCase):
-
     def test_item_meta_classcell_regression(self):
         class MyItem(Item, metaclass=ItemMeta):
             def __init__(self, *args, **kwargs):
diff --git a/tests/test_link.py b/tests/test_link.py
index e0f1efffa..7ba0851ae 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -4,7 +4,6 @@ from scrapy.link import Link
 
 
 class LinkTest(unittest.TestCase):
-
     def _assert_same_links(self, link1, link2):
         self.assertEqual(link1, link2)
         self.assertEqual(hash(link1), hash(link2))
@@ -30,16 +29,26 @@ class LinkTest(unittest.TestCase):
         self._assert_different_links(l4, l5)
         self._assert_same_links(l4, l6)
 
-        l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
-        l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
-        l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
-        l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False)
+        l7 = Link(
+            "http://www.example.com", text="test", fragment="something", nofollow=False
+        )
+        l8 = Link(
+            "http://www.example.com", text="test", fragment="something", nofollow=False
+        )
+        l9 = Link(
+            "http://www.example.com", text="test", fragment="something", nofollow=True
+        )
+        l10 = Link(
+            "http://www.example.com", text="test", fragment="other", nofollow=False
+        )
         self._assert_same_links(l7, l8)
         self._assert_different_links(l7, l9)
         self._assert_different_links(l7, l10)
 
     def test_repr(self):
-        l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
+        l1 = Link(
+            "http://www.example.com", text="test", fragment="something", nofollow=True
+        )
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index e28dc9bdb..520effedb 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -14,108 +14,163 @@ class Base:
         extractor_cls = None
 
         def setUp(self):
-            body = get_testdata('link_extractor', 'linkextractor.html')
-            self.response = HtmlResponse(url='http://example.com/index', body=body)
+            body = get_testdata("link_extractor", "linkextractor.html")
+            self.response = HtmlResponse(url="http://example.com/index", body=body)
 
         def test_urls_type(self):
-            ''' Test that the resulting urls are str objects '''
+            """Test that the resulting urls are str objects"""
             lx = self.extractor_cls()
-            self.assertTrue(all(isinstance(link.url, str)
-                                for link in lx.extract_links(self.response)))
+            self.assertTrue(
+                all(
+                    isinstance(link.url, str)
+                    for link in lx.extract_links(self.response)
+                )
+            )
 
         def test_extract_all_links(self):
             lx = self.extractor_cls()
-            page4_url = 'http://example.com/page%204.html'
+            page4_url = "http://example.com/page%204.html"
 
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=''),
-                Link(url='http://example.com/innertag.html', text='inner tag'),
-                Link(url=page4_url, text='href with whitespaces'),
-            ])
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html#foo",
+                        text="sample 3 repetition with fragment",
+                    ),
+                    Link(url="http://www.google.com/something", text=""),
+                    Link(url="http://example.com/innertag.html", text="inner tag"),
+                    Link(url=page4_url, text="href with whitespaces"),
+                ],
+            )
 
         def test_extract_filter_allow(self):
-            lx = self.extractor_cls(allow=('sample', ))
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
-            ])
+            lx = self.extractor_cls(allow=("sample",))
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html#foo",
+                        text="sample 3 repetition with fragment",
+                    ),
+                ],
+            )
 
         def test_extract_filter_allow_with_duplicates(self):
-            lx = self.extractor_cls(allow=('sample', ), unique=False)
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
-                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
-            ])
+            lx = self.extractor_cls(allow=("sample",), unique=False)
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html",
+                        text="sample 3 repetition",
+                    ),
+                    Link(
+                        url="http://example.com/sample3.html#foo",
+                        text="sample 3 repetition with fragment",
+                    ),
+                ],
+            )
 
         def test_extract_filter_allow_with_duplicates_canonicalize(self):
-            lx = self.extractor_cls(allow=('sample', ), unique=False,
-                                    canonicalize=True)
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
-                Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
-            ])
+            lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True)
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html",
+                        text="sample 3 repetition",
+                    ),
+                    Link(
+                        url="http://example.com/sample3.html",
+                        text="sample 3 repetition with fragment",
+                    ),
+                ],
+            )
 
         def test_extract_filter_allow_no_duplicates_canonicalize(self):
-            lx = self.extractor_cls(allow=('sample',), unique=True,
-                                    canonicalize=True)
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-            ])
+            lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True)
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                ],
+            )
 
         def test_extract_filter_allow_and_deny(self):
-            lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-            ])
+            lx = self.extractor_cls(allow=("sample",), deny=("3",))
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                ],
+            )
 
         def test_extract_filter_allowed_domains(self):
-            lx = self.extractor_cls(allow_domains=('google.com', ))
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=''),
-            ])
+            lx = self.extractor_cls(allow_domains=("google.com",))
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://www.google.com/something", text=""),
+                ],
+            )
 
         def test_extraction_using_single_values(self):
-            '''Test the extractor's behaviour among different situations'''
+            """Test the extractor's behaviour among different situations"""
 
-            lx = self.extractor_cls(allow='sample')
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html#foo',
-                     text='sample 3 repetition with fragment')
-            ])
+            lx = self.extractor_cls(allow="sample")
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html#foo",
+                        text="sample 3 repetition with fragment",
+                    ),
+                ],
+            )
 
-            lx = self.extractor_cls(allow='sample', deny='3')
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-            ])
+            lx = self.extractor_cls(allow="sample", deny="3")
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                ],
+            )
 
-            lx = self.extractor_cls(allow_domains='google.com')
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=''),
-            ])
+            lx = self.extractor_cls(allow_domains="google.com")
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://www.google.com/something", text=""),
+                ],
+            )
 
-            lx = self.extractor_cls(deny_domains='example.com')
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=''),
-            ])
+            lx = self.extractor_cls(deny_domains="example.com")
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://www.google.com/something", text=""),
+                ],
+            )
 
         def test_nofollow(self):
             '''Test the extractor's behaviour for links with rel="nofollow"'''
@@ -141,48 +196,68 @@ class Base:
             response = HtmlResponse("http://example.org/somepage/index.html", body=html)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/about.html', text='About us'),
-                Link(url='http://example.org/follow.html', text='Follow this link'),
-                Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
-                Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
-                Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.org/about.html", text="About us"),
+                    Link(url="http://example.org/follow.html", text="Follow this link"),
+                    Link(
+                        url="http://example.org/nofollow.html",
+                        text="Dont follow this one",
+                        nofollow=True,
+                    ),
+                    Link(
+                        url="http://example.org/nofollow2.html",
+                        text="Choose to follow or not",
+                    ),
+                    Link(
+                        url="http://google.com/something",
+                        text="External link not to follow",
+                        nofollow=True,
+                    ),
+                ],
+            )
 
         def test_matches(self):
-            url1 = 'http://lotsofstuff.com/stuff1/index'
-            url2 = 'http://evenmorestuff.com/uglystuff/index'
+            url1 = "http://lotsofstuff.com/stuff1/index"
+            url2 = "http://evenmorestuff.com/uglystuff/index"
 
-            lx = self.extractor_cls(allow=(r'stuff1', ))
+            lx = self.extractor_cls(allow=(r"stuff1",))
             self.assertEqual(lx.matches(url1), True)
             self.assertEqual(lx.matches(url2), False)
 
-            lx = self.extractor_cls(deny=(r'uglystuff', ))
+            lx = self.extractor_cls(deny=(r"uglystuff",))
             self.assertEqual(lx.matches(url1), True)
             self.assertEqual(lx.matches(url2), False)
 
-            lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
+            lx = self.extractor_cls(allow_domains=("evenmorestuff.com",))
             self.assertEqual(lx.matches(url1), False)
             self.assertEqual(lx.matches(url2), True)
 
-            lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
+            lx = self.extractor_cls(deny_domains=("lotsofstuff.com",))
             self.assertEqual(lx.matches(url1), False)
             self.assertEqual(lx.matches(url2), True)
 
-            lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],
-                                    allow_domains=['blah1.com'],
-                                    deny_domains=['blah2.com'])
-            self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
-            self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
-            self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
-            self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
+            lx = self.extractor_cls(
+                allow=["blah1"],
+                deny=["blah2"],
+                allow_domains=["blah1.com"],
+                deny_domains=["blah2.com"],
+            )
+            self.assertEqual(lx.matches("http://blah1.com/blah1"), True)
+            self.assertEqual(lx.matches("http://blah1.com/blah2"), False)
+            self.assertEqual(lx.matches("http://blah2.com/blah1"), False)
+            self.assertEqual(lx.matches("http://blah2.com/blah2"), False)
 
         def test_restrict_xpaths(self):
-            lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-            ])
+            lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',))
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                ],
+            )
 
         def test_restrict_xpaths_encoding(self):
             """Test restrict_xpaths with encodings"""
@@ -195,84 +270,143 @@ class Base:
             <p><a href="/nofollow.html">This shouldn't be followed</a></p>
             </div>
             </body></html>"""
-            response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
+            response = HtmlResponse(
+                "http://example.org/somepage/index.html",
+                body=html,
+                encoding="windows-1252",
+            )
 
             lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/about.html', text='About us\xa3')])
+            self.assertEqual(
+                lx.extract_links(response),
+                [Link(url="http://example.org/about.html", text="About us\xa3")],
+            )
 
         def test_restrict_xpaths_with_html_entities(self):
             html = b'<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
-            response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
-            links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
-            self.assertEqual(links,
-                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
+            response = HtmlResponse(
+                "http://example.org/somepage/index.html",
+                body=html,
+                encoding="iso8859-15",
+            )
+            links = self.extractor_cls(restrict_xpaths="//p").extract_links(response)
+            self.assertEqual(
+                links, [Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")]
+            )
 
         def test_restrict_xpaths_concat_in_handle_data(self):
             """html entities cause SGMLParser to call handle_data hook twice"""
             body = b"""<html><body><div><a href="/foo">&gt;\xbe\xa9&lt;\xb6\xab</a></body></html>"""
-            response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
+            response = HtmlResponse("http://example.org", body=body, encoding="gb18030")
             lx = self.extractor_cls(restrict_xpaths="//div")
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
-                                   fragment='', nofollow=False)])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="http://example.org/foo",
+                        text=">\u4eac<\u4e1c",
+                        fragment="",
+                        nofollow=False,
+                    )
+                ],
+            )
 
         def test_restrict_css(self):
-            lx = self.extractor_cls(restrict_css=('#subwrapper a',))
-            self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample2.html', text='sample 2')
-            ])
+            lx = self.extractor_cls(restrict_css=("#subwrapper a",))
+            self.assertEqual(
+                lx.extract_links(self.response),
+                [Link(url="http://example.com/sample2.html", text="sample 2")],
+            )
 
         def test_restrict_css_and_restrict_xpaths_together(self):
-            lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
-                                    restrict_css=('#subwrapper + a', ))
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-            ])
+            lx = self.extractor_cls(
+                restrict_xpaths=('//div[@id="subwrapper"]',),
+                restrict_css=("#subwrapper + a",),
+            )
+            self.assertEqual(
+                [link for link in lx.extract_links(self.response)],
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                ],
+            )
 
         def test_area_tag_with_unicode_present(self):
             body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
-            response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
+            response = HtmlResponse("http://example.org", body=body, encoding="utf-8")
             lx = self.extractor_cls()
             lx.extract_links(response)
             lx.extract_links(response)
             lx.extract_links(response)
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text='',
-                                   fragment='', nofollow=False)])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="http://example.org/foo",
+                        text="",
+                        fragment="",
+                        nofollow=False,
+                    )
+                ],
+            )
 
         def test_encoded_url(self):
             body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
-            response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+            response = HtmlResponse(
+                "http://known.fm/AC%2FDC/", body=body, encoding="utf8"
+            )
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="http://known.fm/AC%2FDC/?page=2",
+                        text="BinB",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                ],
+            )
 
         def test_encoded_url_in_restricted_xpath(self):
             body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
-            response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+            response = HtmlResponse(
+                "http://known.fm/AC%2FDC/", body=body, encoding="utf8"
+            )
             lx = self.extractor_cls(restrict_xpaths="//div")
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="http://known.fm/AC%2FDC/?page=2",
+                        text="BinB",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                ],
+            )
 
         def test_ignored_extensions(self):
             # jpg is ignored by default
             html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">"""
             response = HtmlResponse("http://example.org/", body=html)
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/page.html', text='asd'),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.org/page.html", text="asd"),
+                ],
+            )
 
             # override denied extensions
-            lx = self.extractor_cls(deny_extensions=['html'])
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/photo.jpg'),
-            ])
+            lx = self.extractor_cls(deny_extensions=["html"])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.org/photo.jpg"),
+                ],
+            )
 
         def test_process_value(self):
             """Test restrict_xpaths with encodings"""
@@ -280,7 +414,11 @@ class Base:
 <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Text</a>
 <a href="/about.html">About us</a>
             """
-            response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
+            response = HtmlResponse(
+                "http://example.org/somepage/index.html",
+                body=html,
+                encoding="windows-1252",
+            )
 
             def process_value(value):
                 m = re.search(r"javascript:goToPage\('(.*?)'", value)
@@ -288,8 +426,10 @@ class Base:
                     return m.group(1)
 
             lx = self.extractor_cls(process_value=process_value)
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/other/page.html', text='Text')])
+            self.assertEqual(
+                lx.extract_links(response),
+                [Link(url="http://example.org/other/page.html", text="Text")],
+            )
 
         def test_base_url_with_restrict_xpaths(self):
             html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
@@ -297,34 +437,50 @@ class Base:
             </body></html>"""
             response = HtmlResponse("http://example.org/somepage/index.html", body=html)
             lx = self.extractor_cls(restrict_xpaths="//p")
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+            self.assertEqual(
+                lx.extract_links(response),
+                [Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")],
+            )
 
         def test_attrs(self):
             lx = self.extractor_cls(attrs="href")
-            page4_url = 'http://example.com/page%204.html'
+            page4_url = "http://example.com/page%204.html"
 
-            self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=''),
-                Link(url='http://example.com/innertag.html', text='inner tag'),
-                Link(url=page4_url, text='href with whitespaces'),
-            ])
+            self.assertEqual(
+                lx.extract_links(self.response),
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html#foo",
+                        text="sample 3 repetition with fragment",
+                    ),
+                    Link(url="http://www.google.com/something", text=""),
+                    Link(url="http://example.com/innertag.html", text="inner tag"),
+                    Link(url=page4_url, text="href with whitespaces"),
+                ],
+            )
 
-            lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
-            self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=''),
-                Link(url='http://example.com/sample3.html', text='sample 3 text'),
-                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=''),
-                Link(url='http://example.com/innertag.html', text='inner tag'),
-                Link(url=page4_url, text='href with whitespaces'),
-            ])
+            lx = self.extractor_cls(
+                attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()
+            )
+            self.assertEqual(
+                lx.extract_links(self.response),
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample2.jpg", text=""),
+                    Link(url="http://example.com/sample3.html", text="sample 3 text"),
+                    Link(
+                        url="http://example.com/sample3.html#foo",
+                        text="sample 3 repetition with fragment",
+                    ),
+                    Link(url="http://www.google.com/something", text=""),
+                    Link(url="http://example.com/innertag.html", text="inner tag"),
+                    Link(url=page4_url, text="href with whitespaces"),
+                ],
+            )
 
             lx = self.extractor_cls(attrs=None)
             self.assertEqual(lx.extract_links(self.response), [])
@@ -340,26 +496,40 @@ class Base:
             self.assertEqual(lx.extract_links(response), [])
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=''),
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                ],
+            )
 
             lx = self.extractor_cls(tags="area")
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=''),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.com/sample1.html", text=""),
+                ],
+            )
 
             lx = self.extractor_cls(tags="a")
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                ],
+            )
 
-            lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text='sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=''),
-            ])
+            lx = self.extractor_cls(
+                tags=("a", "img"), attrs=("href", "src"), deny_extensions=()
+            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url="http://example.com/sample2.html", text="sample 2"),
+                    Link(url="http://example.com/sample2.jpg", text=""),
+                ],
+            )
 
         def test_tags_attrs(self):
             html = b"""
@@ -370,17 +540,43 @@ class Base:
             """
             response = HtmlResponse("http://example.com/index.html", body=html)
 
-            lx = self.extractor_cls(tags='div', attrs='data-url')
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
-            ])
+            lx = self.extractor_cls(tags="div", attrs="data-url")
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="http://example.com/get?id=1",
+                        text="Item 1",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.com/get?id=2",
+                        text="Item 2",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                ],
+            )
 
-            lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
-            ])
+            lx = self.extractor_cls(tags=("div",), attrs=("data-url",))
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="http://example.com/get?id=1",
+                        text="Item 1",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.com/get?id=2",
+                        text="Item 2",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                ],
+            )
 
         def test_xhtml(self):
             xhtml = b"""
@@ -417,14 +613,36 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
-                         fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
-                         fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
-                ]
+                    Link(
+                        url="http://example.com/about.html",
+                        text="About us",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.com/follow.html",
+                        text="Follow this link",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.com/nofollow.html",
+                        text="Dont follow this one",
+                        fragment="",
+                        nofollow=True,
+                    ),
+                    Link(
+                        url="http://example.com/nofollow2.html",
+                        text="Choose to follow or not",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://google.com/something",
+                        text="External link not to follow",
+                        nofollow=True,
+                    ),
+                ],
             )
 
             response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
@@ -433,14 +651,36 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
-                         fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
-                         fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
-                ]
+                    Link(
+                        url="http://example.com/about.html",
+                        text="About us",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.com/follow.html",
+                        text="Follow this link",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.com/nofollow.html",
+                        text="Dont follow this one",
+                        fragment="",
+                        nofollow=True,
+                    ),
+                    Link(
+                        url="http://example.com/nofollow2.html",
+                        text="Choose to follow or not",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://google.com/something",
+                        text="External link not to follow",
+                        nofollow=True,
+                    ),
+                ],
             )
 
         def test_link_wrong_href(self):
@@ -451,21 +691,42 @@ class Base:
             """
             response = HtmlResponse("http://example.org/index.html", body=html)
             lx = self.extractor_cls()
-            self.assertEqual([link for link in lx.extract_links(response)], [
-                Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
-                Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
-            ])
+            self.assertEqual(
+                [link for link in lx.extract_links(response)],
+                [
+                    Link(
+                        url="http://example.org/item1.html",
+                        text="Item 1",
+                        nofollow=False,
+                    ),
+                    Link(
+                        url="http://example.org/item3.html",
+                        text="Item 3",
+                        nofollow=False,
+                    ),
+                ],
+            )
 
         def test_ftp_links(self):
             body = b"""
             <html><body>
             <div><a href="ftp://www.external.com/">An Item</a></div>
             </body></html>"""
-            response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
+            response = HtmlResponse(
+                "http://www.example.com/index.html", body=body, encoding="utf8"
+            )
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response), [
-                Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
-            ])
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(
+                        url="ftp://www.external.com/",
+                        text="An Item",
+                        fragment="",
+                        nofollow=False,
+                    ),
+                ],
+            )
 
         def test_pickle_extractor(self):
             lx = self.extractor_cls()
@@ -483,10 +744,17 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         """
         response = HtmlResponse("http://example.org/index.html", body=html)
         lx = self.extractor_cls()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
-        ])
+        self.assertEqual(
+            [link for link in lx.extract_links(response)],
+            [
+                Link(
+                    url="http://example.org/item1.html", text="Item 1", nofollow=False
+                ),
+                Link(
+                    url="http://example.org/item3.html", text="Item 3", nofollow=False
+                ),
+            ],
+        )
 
     def test_link_restrict_text(self):
         html = b"""
@@ -496,21 +764,46 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         """
         response = HtmlResponse("http://example.org/index.html", body=html)
         # Simple text inclusion test
-        lx = self.extractor_cls(restrict_text='dog')
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
-        ])
+        lx = self.extractor_cls(restrict_text="dog")
+        self.assertEqual(
+            [link for link in lx.extract_links(response)],
+            [
+                Link(
+                    url="http://example.org/item2.html",
+                    text="Pic of a dog",
+                    nofollow=False,
+                ),
+            ],
+        )
         # Unique regex test
-        lx = self.extractor_cls(restrict_text=r'of.*dog')
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
-        ])
+        lx = self.extractor_cls(restrict_text=r"of.*dog")
+        self.assertEqual(
+            [link for link in lx.extract_links(response)],
+            [
+                Link(
+                    url="http://example.org/item2.html",
+                    text="Pic of a dog",
+                    nofollow=False,
+                ),
+            ],
+        )
         # Multiple regex test
-        lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
-            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
-        ])
+        lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"])
+        self.assertEqual(
+            [link for link in lx.extract_links(response)],
+            [
+                Link(
+                    url="http://example.org/item1.html",
+                    text="Pic of a cat",
+                    nofollow=False,
+                ),
+                Link(
+                    url="http://example.org/item2.html",
+                    text="Pic of a dog",
+                    nofollow=False,
+                ),
+            ],
+        )
 
     def test_restrict_xpaths_with_html_entities(self):
         super().test_restrict_xpaths_with_html_entities()
diff --git a/tests/test_loader.py b/tests/test_loader.py
index b3e44d36b..9dd298864 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -59,32 +59,31 @@ class DefaultedItemLoader(NameItemLoader):
 
 # test processors
 def processor_with_args(value, other=None, loader_context=None):
-    if 'key' in loader_context:
-        return loader_context['key']
+    if "key" in loader_context:
+        return loader_context["key"]
     return value
 
 
 class BasicItemLoaderTest(unittest.TestCase):
-
     def test_add_value_on_unknown_field(self):
         il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
+        self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = 'lala'
+        i["summary"] = "lala"
         il = ItemLoader(item=i)
-        il.add_value('name', 'marta')
+        il.add_value("name", "marta")
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], ['lala'])
-        self.assertEqual(item['name'], ['marta'])
+        self.assertEqual(item["summary"], ["lala"])
+        self.assertEqual(item["name"], ["marta"])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', 'marta')
+        il.add_value("name", "marta")
         item = il.load_item()
-        self.assertEqual(item['name'], ['Marta'])
+        self.assertEqual(item["name"], ["Marta"])
 
 
 class InitializationTestMixin:
@@ -93,85 +92,94 @@ class InitializationTestMixin:
 
     def test_keep_single_value(self):
         """Loaded item should contain values from the initial item"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]})
 
     def test_keep_list(self):
         """Loaded item should contain values from the initial item"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
 
     def test_add_value_singlevalue_singlevalue(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        il.add_value('name', 'bar')
+        il.add_value("name", "bar")
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
 
     def test_add_value_singlevalue_list(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        il.add_value('name', ['item', 'loader'])
+        il.add_value("name", ["item", "loader"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']})
+        self.assertEqual(
+            ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]}
+        )
 
     def test_add_value_list_singlevalue(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        il.add_value('name', 'qwerty')
+        il.add_value("name", "qwerty")
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']})
+        self.assertEqual(
+            ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]}
+        )
 
     def test_add_value_list_list(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        il.add_value('name', ['item', 'loader'])
+        il.add_value("name", ["item", "loader"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']})
+        self.assertEqual(
+            ItemAdapter(loaded_item).asdict(),
+            {"name": ["foo", "bar", "item", "loader"]},
+        )
 
     def test_get_output_value_singlevalue(self):
         """Getting output value must not remove value from item"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        self.assertEqual(il.get_output_value('name'), ['foo'])
+        self.assertEqual(il.get_output_value("name"), ["foo"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']}))
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]}))
 
     def test_get_output_value_list(self):
         """Getting output value must not remove value from item"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+        self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']}))
+        self.assertEqual(
+            ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]})
+        )
 
     def test_values_single(self):
         """Values from initial item must be added to loader._values"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        self.assertEqual(il._values.get('name'), ['foo'])
+        self.assertEqual(il._values.get("name"), ["foo"])
 
     def test_values_list(self):
         """Values from initial item must be added to loader._values"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+        self.assertEqual(il._values.get("name"), ["foo", "bar"])
 
 
 class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase):
@@ -207,42 +215,52 @@ class NoInputReprocessingFromItemTest(unittest.TestCase):
     """
     Loaders initialized from loaded items must not reprocess fields (Item instances)
     """
+
     def test_avoid_reprocessing_with_initial_values_single(self):
-        il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo'))
+        il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo"))
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, {'title': 'foo'})
-        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
+        self.assertEqual(il_loaded, {"title": "foo"})
+        self.assertEqual(
+            NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"}
+        )
 
     def test_avoid_reprocessing_with_initial_values_list(self):
-        il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title=['foo', 'bar']))
+        il = NoInputReprocessingItemLoader(
+            item=NoInputReprocessingItem(title=["foo", "bar"])
+        )
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, {'title': 'foo'})
-        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
+        self.assertEqual(il_loaded, {"title": "foo"})
+        self.assertEqual(
+            NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"}
+        )
 
     def test_avoid_reprocessing_without_initial_values_single(self):
         il = NoInputReprocessingItemLoader()
-        il.add_value('title', 'FOO')
+        il.add_value("title", "FOO")
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, {'title': 'FOO'})
-        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
+        self.assertEqual(il_loaded, {"title": "FOO"})
+        self.assertEqual(
+            NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"}
+        )
 
     def test_avoid_reprocessing_without_initial_values_list(self):
         il = NoInputReprocessingItemLoader()
-        il.add_value('title', ['foo', 'bar'])
+        il.add_value("title", ["foo", "bar"])
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, {'title': 'FOO'})
-        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
+        self.assertEqual(il_loaded, {"title": "FOO"})
+        self.assertEqual(
+            NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"}
+        )
 
 
 class TestOutputProcessorItem(unittest.TestCase):
     def test_output_processor(self):
-
         class TempItem(Item):
             temp = Field()
 
             def __init__(self, *args, **kwargs):
                 super().__init__(self, *args, **kwargs)
-                self.setdefault('temp', 0.3)
+                self.setdefault("temp", 0.3)
 
         class TempLoader(ItemLoader):
             default_item_class = TempItem
@@ -252,11 +270,14 @@ class TestOutputProcessorItem(unittest.TestCase):
         loader = TempLoader()
         item = loader.load_item()
         self.assertIsInstance(item, TempItem)
-        self.assertEqual(dict(item), {'temp': 0.3})
+        self.assertEqual(dict(item), {"temp": 0.3})
 
 
 class SelectortemLoaderTest(unittest.TestCase):
-    response = HtmlResponse(url="", encoding='utf-8', body=b"""
+    response = HtmlResponse(
+        url="",
+        encoding="utf-8",
+        body=b"""
     <html>
     <body>
     <div id="id">marta</div>
@@ -265,7 +286,8 @@ class SelectortemLoaderTest(unittest.TestCase):
     <img src="/images/logo.png" width="244" height="65" alt="Scrapy">
     </body>
     </html>
-    """)
+    """,
+    )
 
     def test_init_method(self):
         l = TestItemLoader()
@@ -273,28 +295,28 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_init_method_errors(self):
         l = TestItemLoader()
-        self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href')
-        self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href')
-        self.assertRaises(RuntimeError, l.get_xpath, '//a/@href')
-        self.assertRaises(RuntimeError, l.add_css, 'name', '#name::text')
-        self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text')
-        self.assertRaises(RuntimeError, l.get_css, '#name::text')
+        self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")
+        self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")
+        self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")
+        self.assertRaises(RuntimeError, l.add_css, "name", "#name::text")
+        self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text")
+        self.assertRaises(RuntimeError, l.get_css, "#name::text")
 
     def test_init_method_with_selector(self):
         sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
-        l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
+        l.add_xpath("name", "//div/text()")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
 
     def test_init_method_with_selector_css(self):
         sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
-        l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
+        l.add_css("name", "div::text")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
 
     def test_init_method_with_base_response(self):
         """Selector should be None after initialization"""
@@ -306,122 +328,131 @@ class SelectortemLoaderTest(unittest.TestCase):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
-        l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
+        l.add_xpath("name", "//div/text()")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
 
     def test_init_method_with_response_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
-        l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
+        l.add_css("name", "div::text")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
 
-        l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
+        l.add_css("url", "a::attr(href)")
+        self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
 
         # combining/accumulating CSS selectors and XPath expressions
-        l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
+        l.add_xpath("name", "//div/text()")
+        self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"])
 
-        l.add_xpath('url', '//img/@src')
-        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
+        l.add_xpath("url", "//img/@src")
+        self.assertEqual(
+            l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"]
+        )
 
     def test_add_xpath_re(self):
         l = TestItemLoader(response=self.response)
-        l.add_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), ['Ma'])
+        l.add_xpath("name", "//div/text()", re="ma")
+        self.assertEqual(l.get_output_value("name"), ["Ma"])
 
     def test_replace_xpath(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
-        l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
-        l.replace_xpath('name', '//p/text()')
-        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+        l.add_xpath("name", "//div/text()")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
+        l.replace_xpath("name", "//p/text()")
+        self.assertEqual(l.get_output_value("name"), ["Paragraph"])
 
-        l.replace_xpath('name', ['//p/text()', '//div/text()'])
-        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
+        l.replace_xpath("name", ["//p/text()", "//div/text()"])
+        self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
 
     def test_get_xpath(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
+        self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"])
+        self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph")
+        self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa")
 
-        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
+        self.assertEqual(
+            l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"]
+        )
 
     def test_replace_xpath_multi_fields(self):
         l = TestItemLoader(response=self.response)
-        l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
-        l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+        l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x})
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
+        l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x})
+        self.assertEqual(l.get_output_value("name"), ["Paragraph"])
 
     def test_replace_xpath_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
-        l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
-        l.replace_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), ['Ma'])
+        l.add_xpath("name", "//div/text()")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
+        l.replace_xpath("name", "//div/text()", re="ma")
+        self.assertEqual(l.get_output_value("name"), ["Ma"])
 
     def test_add_css_re(self):
         l = TestItemLoader(response=self.response)
-        l.add_css('name', 'div::text', re='ma')
-        self.assertEqual(l.get_output_value('name'), ['Ma'])
+        l.add_css("name", "div::text", re="ma")
+        self.assertEqual(l.get_output_value("name"), ["Ma"])
 
-        l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
+        l.add_css("url", "a::attr(href)", re="http://(.+)")
+        self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
 
     def test_replace_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
-        l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
-        l.replace_css('name', 'p::text')
-        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+        l.add_css("name", "div::text")
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
+        l.replace_css("name", "p::text")
+        self.assertEqual(l.get_output_value("name"), ["Paragraph"])
 
-        l.replace_css('name', ['p::text', 'div::text'])
-        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
+        l.replace_css("name", ["p::text", "div::text"])
+        self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
 
-        l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
-        l.replace_css('url', 'img::attr(src)')
-        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
+        l.add_css("url", "a::attr(href)", re="http://(.+)")
+        self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
+        l.replace_css("url", "img::attr(src)")
+        self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
 
     def test_get_css(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_css('p::text'), ['paragraph'])
-        self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
-        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
+        self.assertEqual(l.get_css("p::text"), ["paragraph"])
+        self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph")
+        self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa")
 
-        self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
-        self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
-                         ['http://www.scrapy.org', '/images/logo.png'])
+        self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"])
+        self.assertEqual(
+            l.get_css(["a::attr(href)", "img::attr(src)"]),
+            ["http://www.scrapy.org", "/images/logo.png"],
+        )
 
     def test_replace_css_multi_fields(self):
         l = TestItemLoader(response=self.response)
-        l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), ['Marta'])
-        l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+        l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x})
+        self.assertEqual(l.get_output_value("name"), ["Marta"])
+        l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x})
+        self.assertEqual(l.get_output_value("name"), ["Paragraph"])
 
-        l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
-        l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
+        l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x})
+        self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
+        l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x})
+        self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
 
     def test_replace_css_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
-        l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
-        l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
-        self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
+        l.add_css("url", "a::attr(href)")
+        self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
+        l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)")
+        self.assertEqual(l.get_output_value("url"), ["scrapy.org"])
 
 
 class SubselectorLoaderTest(unittest.TestCase):
-    response = HtmlResponse(url="", encoding='utf-8', body=b"""
+    response = HtmlResponse(
+        url="",
+        encoding="utf-8",
+        body=b"""
     <html>
     <body>
     <header>
@@ -434,76 +465,88 @@ class SubselectorLoaderTest(unittest.TestCase):
     </footer>
     </body>
     </html>
-    """)
+    """,
+    )
 
     def test_nested_xpath(self):
         l = NestedItemLoader(response=self.response)
 
         nl = l.nested_xpath("//header")
-        nl.add_xpath('name', 'div/text()')
-        nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
+        nl.add_xpath("name", "div/text()")
+        nl.add_css("name_div", "#id")
+        nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), ['marta'])
-        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), ['marta'])
+        self.assertEqual(l.get_output_value("name"), ["marta"])
+        self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value("name_value"), ["marta"])
 
-        self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
-        self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
-        self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+        self.assertEqual(l.get_output_value("name"), nl.get_output_value("name"))
+        self.assertEqual(
+            l.get_output_value("name_div"), nl.get_output_value("name_div")
+        )
+        self.assertEqual(
+            l.get_output_value("name_value"), nl.get_output_value("name_value")
+        )
 
     def test_nested_css(self):
         l = NestedItemLoader(response=self.response)
         nl = l.nested_css("header")
-        nl.add_xpath('name', 'div/text()')
-        nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
+        nl.add_xpath("name", "div/text()")
+        nl.add_css("name_div", "#id")
+        nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), ['marta'])
-        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), ['marta'])
+        self.assertEqual(l.get_output_value("name"), ["marta"])
+        self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value("name_value"), ["marta"])
 
-        self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
-        self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
-        self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+        self.assertEqual(l.get_output_value("name"), nl.get_output_value("name"))
+        self.assertEqual(
+            l.get_output_value("name_div"), nl.get_output_value("name_div")
+        )
+        self.assertEqual(
+            l.get_output_value("name_value"), nl.get_output_value("name_value")
+        )
 
     def test_nested_replace(self):
         l = NestedItemLoader(response=self.response)
-        nl1 = l.nested_xpath('//footer')
-        nl2 = nl1.nested_xpath('a')
+        nl1 = l.nested_xpath("//footer")
+        nl2 = nl1.nested_xpath("a")
 
-        l.add_xpath('url', '//footer/a/@href')
-        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
-        nl1.replace_xpath('url', 'img/@src')
-        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
-        nl2.replace_xpath('url', '@href')
-        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
+        l.add_xpath("url", "//footer/a/@href")
+        self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
+        nl1.replace_xpath("url", "img/@src")
+        self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
+        nl2.replace_xpath("url", "@href")
+        self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
 
     def test_nested_ordering(self):
         l = NestedItemLoader(response=self.response)
-        nl1 = l.nested_xpath('//footer')
-        nl2 = nl1.nested_xpath('a')
+        nl1 = l.nested_xpath("//footer")
+        nl2 = nl1.nested_xpath("a")
 
-        nl1.add_xpath('url', 'img/@src')
-        l.add_xpath('url', '//footer/a/@href')
-        nl2.add_xpath('url', 'text()')
-        l.add_xpath('url', '//footer/a/@href')
+        nl1.add_xpath("url", "img/@src")
+        l.add_xpath("url", "//footer/a/@href")
+        nl2.add_xpath("url", "text()")
+        l.add_xpath("url", "//footer/a/@href")
 
-        self.assertEqual(l.get_output_value('url'), [
-            '/images/logo.png',
-            'http://www.scrapy.org',
-            'homepage',
-            'http://www.scrapy.org',
-        ])
+        self.assertEqual(
+            l.get_output_value("url"),
+            [
+                "/images/logo.png",
+                "http://www.scrapy.org",
+                "homepage",
+                "http://www.scrapy.org",
+            ],
+        )
 
     def test_nested_load_item(self):
         l = NestedItemLoader(response=self.response)
-        nl1 = l.nested_xpath('//footer')
-        nl2 = nl1.nested_xpath('img')
+        nl1 = l.nested_xpath("//footer")
+        nl2 = nl1.nested_xpath("img")
 
-        l.add_xpath('name', '//header/div/text()')
-        nl1.add_xpath('url', 'a/@href')
-        nl2.add_xpath('image', '@src')
+        l.add_xpath("name", "//header/div/text()")
+        nl1.add_xpath("url", "a/@href")
+        nl2.add_xpath("image", "@src")
 
         item = l.load_item()
 
@@ -511,13 +554,14 @@ class SubselectorLoaderTest(unittest.TestCase):
         assert item is nl1.item
         assert item is nl2.item
 
-        self.assertEqual(item['name'], ['marta'])
-        self.assertEqual(item['url'], ['http://www.scrapy.org'])
-        self.assertEqual(item['image'], ['/images/logo.png'])
+        self.assertEqual(item["name"], ["marta"])
+        self.assertEqual(item["url"], ["http://www.scrapy.org"])
+        self.assertEqual(item["image"], ["/images/logo.png"])
 
 
 # Functions as processors
 
+
 def function_processor_strip(iterable):
     return [x.strip() for x in iterable]
 
@@ -538,15 +582,11 @@ class FunctionProcessorItemLoader(ItemLoader):
 
 
 class FunctionProcessorTestCase(unittest.TestCase):
-
     def test_processor_defined_in_item(self):
         lo = FunctionProcessorItemLoader()
-        lo.add_value('foo', '  bar  ')
-        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
-        self.assertEqual(
-            dict(lo.load_item()),
-            {'foo': ['BAR', 'ASDF', 'QWERTY']}
-        )
+        lo.add_value("foo", "  bar  ")
+        lo.add_value("foo", ["  asdf  ", "  qwerty  "])
+        self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
 
 
 if __name__ == "__main__":
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 14b3b5568..8757db0ce 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -7,8 +7,14 @@ import unittest
 import warnings
 from functools import partial
 
-from itemloaders.processors import (Compose, Identity, Join,
-                                    MapCompose, SelectJmes, TakeFirst)
+from itemloaders.processors import (
+    Compose,
+    Identity,
+    Join,
+    MapCompose,
+    SelectJmes,
+    TakeFirst,
+)
 
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
@@ -42,28 +48,27 @@ class DefaultedItemLoader(NameItemLoader):
 
 # test processors
 def processor_with_args(value, other=None, loader_context=None):
-    if 'key' in loader_context:
-        return loader_context['key']
+    if "key" in loader_context:
+        return loader_context["key"]
     return value
 
 
 class BasicItemLoaderTest(unittest.TestCase):
-
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = 'lala'
+        i["summary"] = "lala"
         il = ItemLoader(item=i)
-        il.add_value('name', 'marta')
+        il.add_value("name", "marta")
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], ['lala'])
-        self.assertEqual(item['name'], ['marta'])
+        self.assertEqual(item["summary"], ["lala"])
+        self.assertEqual(item["name"], ["marta"])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', 'marta')
+        il.add_value("name", "marta")
         item = il.load_item()
-        self.assertEqual(item['name'], ['Marta'])
+        self.assertEqual(item["name"], ["Marta"])
 
     def test_load_item_ignore_none_field_values(self):
         def validate_sku(value):
@@ -76,206 +81,223 @@ class BasicItemLoaderTest(unittest.TestCase):
             price_out = Compose(TakeFirst(), float)
             sku_out = Compose(TakeFirst(), validate_sku)
 
-        valid_fragment = 'SKU: 1234'
-        invalid_fragment = 'SKU: not available'
-        sku_re = 'SKU: (.+)'
+        valid_fragment = "SKU: 1234"
+        invalid_fragment = "SKU: not available"
+        sku_re = "SKU: (.+)"
 
         il = MyLoader(item={})
         # Should not return "sku: None".
-        il.add_value('sku', [invalid_fragment], re=sku_re)
+        il.add_value("sku", [invalid_fragment], re=sku_re)
         # Should not ignore empty values.
-        il.add_value('name', '')
-        il.add_value('price', ['0'])
-        self.assertEqual(il.load_item(), {
-            'name': '',
-            'price': 0.0,
-        })
+        il.add_value("name", "")
+        il.add_value("price", ["0"])
+        self.assertEqual(
+            il.load_item(),
+            {
+                "name": "",
+                "price": 0.0,
+            },
+        )
 
-        il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], '1234')
+        il.replace_value("sku", [valid_fragment], re=sku_re)
+        self.assertEqual(il.load_item()["sku"], "1234")
 
     def test_self_referencing_loader(self):
         class MyLoader(ItemLoader):
             url_out = TakeFirst()
 
             def img_url_out(self, values):
-                return (self.get_output_value('url') or '') + values[0]
+                return (self.get_output_value("url") or "") + values[0]
 
         il = MyLoader(item={})
-        il.add_value('url', 'http://example.com/')
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'url': 'http://example.com/',
-            'img_url': 'http://example.com/1234.png',
-        })
+        il.add_value("url", "http://example.com/")
+        il.add_value("img_url", "1234.png")
+        self.assertEqual(
+            il.load_item(),
+            {
+                "url": "http://example.com/",
+                "img_url": "http://example.com/1234.png",
+            },
+        )
 
         il = MyLoader(item={})
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'img_url': '1234.png',
-        })
+        il.add_value("img_url", "1234.png")
+        self.assertEqual(
+            il.load_item(),
+            {
+                "img_url": "1234.png",
+            },
+        )
 
     def test_add_value(self):
         il = TestItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_collected_values('name'), ['Marta'])
-        self.assertEqual(il.get_output_value('name'), ['Marta'])
-        il.add_value('name', 'pepe')
-        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
-        self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_collected_values("name"), ["Marta"])
+        self.assertEqual(il.get_output_value("name"), ["Marta"])
+        il.add_value("name", "pepe")
+        self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"])
+        self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"])
 
         # test add object value
-        il.add_value('summary', {'key': 1})
-        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
+        il.add_value("summary", {"key": 1})
+        self.assertEqual(il.get_collected_values("summary"), [{"key": 1}])
 
-        il.add_value(None, 'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
+        il.add_value(None, "Jim", lambda x: {"name": x})
+        self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"])
 
     def test_add_zero(self):
         il = NameItemLoader()
-        il.add_value('name', 0)
-        self.assertEqual(il.get_collected_values('name'), [0])
+        il.add_value("name", 0)
+        self.assertEqual(il.get_collected_values("name"), [0])
 
     def test_replace_value(self):
         il = TestItemLoader()
-        il.replace_value('name', 'marta')
-        self.assertEqual(il.get_collected_values('name'), ['Marta'])
-        self.assertEqual(il.get_output_value('name'), ['Marta'])
-        il.replace_value('name', 'pepe')
-        self.assertEqual(il.get_collected_values('name'), ['Pepe'])
-        self.assertEqual(il.get_output_value('name'), ['Pepe'])
+        il.replace_value("name", "marta")
+        self.assertEqual(il.get_collected_values("name"), ["Marta"])
+        self.assertEqual(il.get_output_value("name"), ["Marta"])
+        il.replace_value("name", "pepe")
+        self.assertEqual(il.get_collected_values("name"), ["Pepe"])
+        self.assertEqual(il.get_output_value("name"), ["Pepe"])
 
-        il.replace_value(None, 'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), ['Jim'])
+        il.replace_value(None, "Jim", lambda x: {"name": x})
+        self.assertEqual(il.get_collected_values("name"), ["Jim"])
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
-        self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
-        self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
+        self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper))
+        self.assertEqual(
+            ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$")
+        )
+        self.assertEqual(
+            "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$")
+        )
 
-        il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
-        self.assertEqual(['foo'], il.get_collected_values('name'))
-        il.replace_value('name', 'name:bar', re='name:(.*)$')
-        self.assertEqual(['bar'], il.get_collected_values('name'))
+        il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$")
+        self.assertEqual(["foo"], il.get_collected_values("name"))
+        il.replace_value("name", "name:bar", re="name:(.*)$")
+        self.assertEqual(["bar"], il.get_collected_values("name"))
 
     def test_iter_on_input_processor_input(self):
         class NameFirstItemLoader(NameItemLoader):
             name_in = TakeFirst()
 
         il = NameFirstItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_collected_values('name'), ['marta'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_collected_values("name"), ["marta"])
         il = NameFirstItemLoader()
-        il.add_value('name', ['marta', 'jose'])
-        self.assertEqual(il.get_collected_values('name'), ['marta'])
+        il.add_value("name", ["marta", "jose"])
+        self.assertEqual(il.get_collected_values("name"), ["marta"])
 
         il = NameFirstItemLoader()
-        il.replace_value('name', 'marta')
-        self.assertEqual(il.get_collected_values('name'), ['marta'])
+        il.replace_value("name", "marta")
+        self.assertEqual(il.get_collected_values("name"), ["marta"])
         il = NameFirstItemLoader()
-        il.replace_value('name', ['marta', 'jose'])
-        self.assertEqual(il.get_collected_values('name'), ['marta'])
+        il.replace_value("name", ["marta", "jose"])
+        self.assertEqual(il.get_collected_values("name"), ["marta"])
 
         il = NameFirstItemLoader()
-        il.add_value('name', 'marta')
-        il.add_value('name', ['jose', 'pedro'])
-        self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
+        il.add_value("name", "marta")
+        il.add_value("name", ["jose", "pedro"])
+        self.assertEqual(il.get_collected_values("name"), ["marta", "jose"])
 
     def test_map_compose_filter(self):
         def filter_world(x):
-            return None if x == 'world' else x
+            return None if x == "world" else x
 
         proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
-                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+        self.assertEqual(
+            proc(["hello", "world", "this", "is", "scrapy"]),
+            ["HELLO", "THIS", "IS", "SCRAPY"],
+        )
 
     def test_map_compose_filter_multil(self):
         class TestItemLoader(NameItemLoader):
             name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['Mart'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["Mart"])
         item = il.load_item()
-        self.assertEqual(item['name'], ['Mart'])
+        self.assertEqual(item["name"], ["Mart"])
 
     def test_default_input_processor(self):
         il = DefaultedItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['mart'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["mart"])
 
     def test_inherited_default_input_processor(self):
         class InheritDefaultedItemLoader(DefaultedItemLoader):
             pass
 
         il = InheritDefaultedItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['mart'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["mart"])
 
     def test_input_processor_inheritance(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(lambda v: v.lower())
 
         il = ChildItemLoader()
-        il.add_value('url', 'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.add_value("url", "HTTP://scrapy.ORG")
+        self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["Marta"])
 
         class ChildChildItemLoader(ChildItemLoader):
             url_in = MapCompose(lambda v: v.upper())
             summary_in = MapCompose(lambda v: v)
 
         il = ChildChildItemLoader()
-        il.add_value('url', 'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.add_value("url", "http://scrapy.org")
+        self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["Marta"])
 
     def test_empty_map_compose(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['marta'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["marta"])
 
     def test_identity_input_processor(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = Identity()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['marta'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["marta"])
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
             name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['mARTA'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["mARTA"])
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
+            name_in = MapCompose(
+                DefaultedItemLoader.default_input_processor, str.swapcase
+            )
 
         il = ChildDefaultedItemLoader()
-        il.add_value('name', 'marta')
-        self.assertEqual(il.get_output_value('name'), ['MART'])
+        il.add_value("name", "marta")
+        self.assertEqual(il.get_output_value("name"), ["MART"])
 
     def test_output_processor_using_function(self):
         il = TestItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = " ".join
 
         il = TakeFirstItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), "Mar Ta")
 
     def test_output_processor_error(self):
         class TestItemLoader(ItemLoader):
@@ -283,9 +305,9 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = MapCompose(float)
 
         il = TestItemLoader()
-        il.add_value('name', ['$10'])
+        il.add_value("name", ["$10"])
         try:
-            float('$10')
+            float("$10")
         except Exception as e:
             expected_exc_str = str(e)
 
@@ -296,118 +318,118 @@ class BasicItemLoaderTest(unittest.TestCase):
             exc = e
         assert isinstance(exc, ValueError)
         s = str(exc)
-        assert 'name' in s, s
-        assert '$10' in s, s
-        assert 'ValueError' in s, s
+        assert "name" in s, s
+        assert "$10" in s, s
+        assert "ValueError" in s, s
         assert expected_exc_str in s, s
 
     def test_output_processor_using_classes(self):
         il = TestItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join()
 
         il = TakeFirstItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), "Mar Ta")
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join("<br>")
 
         il = TakeFirstItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")
 
     def test_default_output_processor(self):
         il = TestItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
         class LalaItemLoader(TestItemLoader):
             default_output_processor = Identity()
 
         il = LalaItemLoader()
-        il.add_value('name', ['mar', 'ta'])
-        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+        il.add_value("name", ["mar", "ta"])
+        self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
 
     def test_loader_context_on_declaration(self):
         class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key='val')
+            url_in = MapCompose(processor_with_args, key="val")
 
         il = ChildItemLoader()
-        il.add_value('url', 'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', 'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.add_value("url", "text")
+        self.assertEqual(il.get_output_value("url"), ["val"])
+        il.replace_value("url", "text2")
+        self.assertEqual(il.get_output_value("url"), ["val"])
 
     def test_loader_context_on_instantiation(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(processor_with_args)
 
-        il = ChildItemLoader(key='val')
-        il.add_value('url', 'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', 'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
+        il = ChildItemLoader(key="val")
+        il.add_value("url", "text")
+        self.assertEqual(il.get_output_value("url"), ["val"])
+        il.replace_value("url", "text2")
+        self.assertEqual(il.get_output_value("url"), ["val"])
 
     def test_loader_context_on_assign(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(processor_with_args)
 
         il = ChildItemLoader()
-        il.context['key'] = 'val'
-        il.add_value('url', 'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', 'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.context["key"] = "val"
+        il.add_value("url", "text")
+        self.assertEqual(il.get_output_value("url"), ["val"])
+        il.replace_value("url", "text2")
+        self.assertEqual(il.get_output_value("url"), ["val"])
 
     def test_item_passed_to_input_processor_functions(self):
         def processor(value, loader_context):
-            return loader_context['item']['name']
+            return loader_context["item"]["name"]
 
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(processor)
 
-        it = TestItem(name='marta')
+        it = TestItem(name="marta")
         il = ChildItemLoader(item=it)
-        il.add_value('url', 'text')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', 'text2')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
+        il.add_value("url", "text")
+        self.assertEqual(il.get_output_value("url"), ["marta"])
+        il.replace_value("url", "text2")
+        self.assertEqual(il.get_output_value("url"), ["marta"])
 
     def test_compose_processor(self):
         class TestItemLoader(NameItemLoader):
             name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', ['marta', 'other'])
-        self.assertEqual(il.get_output_value('name'), 'Mart')
+        il.add_value("name", ["marta", "other"])
+        self.assertEqual(il.get_output_value("name"), "Mart")
         item = il.load_item()
-        self.assertEqual(item['name'], 'Mart')
+        self.assertEqual(item["name"], "Mart")
 
     def test_partial_processor(self):
         def join(values, sep=None, loader_context=None, ignored=None):
             if sep is not None:
                 return sep.join(values)
-            if loader_context and 'sep' in loader_context:
-                return loader_context['sep'].join(values)
-            return ''.join(values)
+            if loader_context and "sep" in loader_context:
+                return loader_context["sep"].join(values)
+            return "".join(values)
 
         class TestItemLoader(NameItemLoader):
-            name_out = Compose(partial(join, sep='+'))
-            url_out = Compose(partial(join, loader_context={'sep': '.'}))
-            summary_out = Compose(partial(join, ignored='foo'))
+            name_out = Compose(partial(join, sep="+"))
+            url_out = Compose(partial(join, loader_context={"sep": "."}))
+            summary_out = Compose(partial(join, ignored="foo"))
 
         il = TestItemLoader()
-        il.add_value('name', ['rabbit', 'hole'])
-        il.add_value('url', ['rabbit', 'hole'])
-        il.add_value('summary', ['rabbit', 'hole'])
+        il.add_value("name", ["rabbit", "hole"])
+        il.add_value("url", ["rabbit", "hole"])
+        il.add_value("summary", ["rabbit", "hole"])
         item = il.load_item()
-        self.assertEqual(item['name'], 'rabbit+hole')
-        self.assertEqual(item['url'], 'rabbit.hole')
-        self.assertEqual(item['summary'], 'rabbithole')
+        self.assertEqual(item["name"], "rabbit+hole")
+        self.assertEqual(item["url"], "rabbit.hole")
+        self.assertEqual(item["summary"], "rabbithole")
 
     def test_error_input_processor(self):
         class TestItem(Item):
@@ -418,8 +440,7 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_in = MapCompose(float)
 
         il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          ['marta', 'other'])
+        self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"])
 
     def test_error_output_processor(self):
         class TestItem(Item):
@@ -430,7 +451,7 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(Join(), float)
 
         il = TestItemLoader()
-        il.add_value('name', 'marta')
+        il.add_value("name", "marta")
         with self.assertRaises(ValueError):
             il.load_item()
 
@@ -442,8 +463,9 @@ class BasicItemLoaderTest(unittest.TestCase):
             default_item_class = TestItem
 
         il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          ['marta', 'other'], Compose(float))
+        self.assertRaises(
+            ValueError, il.add_value, "name", ["marta", "other"], Compose(float)
+        )
 
 
 class InitializationFromDictTest(unittest.TestCase):
@@ -452,85 +474,85 @@ class InitializationFromDictTest(unittest.TestCase):
 
     def test_keep_single_value(self):
         """Loaded item should contain values from the initial item"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+        self.assertEqual(dict(loaded_item), {"name": ["foo"]})
 
     def test_keep_list(self):
         """Loaded item should contain values from the initial item"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+        self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]})
 
     def test_add_value_singlevalue_singlevalue(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        il.add_value('name', 'bar')
+        il.add_value("name", "bar")
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+        self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]})
 
     def test_add_value_singlevalue_list(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        il.add_value('name', ['item', 'loader'])
+        il.add_value("name", ["item", "loader"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+        self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]})
 
     def test_add_value_list_singlevalue(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        il.add_value('name', 'qwerty')
+        il.add_value("name", "qwerty")
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+        self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]})
 
     def test_add_value_list_list(self):
         """Values added after initialization should be appended"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        il.add_value('name', ['item', 'loader'])
+        il.add_value("name", ["item", "loader"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+        self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]})
 
     def test_get_output_value_singlevalue(self):
         """Getting output value must not remove value from item"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        self.assertEqual(il.get_output_value('name'), ['foo'])
+        self.assertEqual(il.get_output_value("name"), ["foo"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+        self.assertEqual(loaded_item, dict({"name": ["foo"]}))
 
     def test_get_output_value_list(self):
         """Getting output value must not remove value from item"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+        self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+        self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]}))
 
     def test_values_single(self):
         """Values from initial item must be added to loader._values"""
-        input_item = self.item_class(name='foo')
+        input_item = self.item_class(name="foo")
         il = ItemLoader(item=input_item)
-        self.assertEqual(il._values.get('name'), ['foo'])
+        self.assertEqual(il._values.get("name"), ["foo"])
 
     def test_values_list(self):
         """Values from initial item must be added to loader._values"""
-        input_item = self.item_class(name=['foo', 'bar'])
+        input_item = self.item_class(name=["foo", "bar"])
         il = ItemLoader(item=input_item)
-        self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+        self.assertEqual(il._values.get("name"), ["foo", "bar"])
 
 
 class BaseNoInputReprocessingLoader(ItemLoader):
@@ -546,40 +568,48 @@ class NoInputReprocessingFromDictTest(unittest.TestCase):
     """
     Loaders initialized from loaded items must not reprocess fields (dict instances)
     """
+
     def test_avoid_reprocessing_with_initial_values_single(self):
-        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
+        il = NoInputReprocessingDictLoader(item=dict(title="foo"))
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+        self.assertEqual(il_loaded, dict(title="foo"))
+        self.assertEqual(
+            NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo")
+        )
 
     def test_avoid_reprocessing_with_initial_values_list(self):
-        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
+        il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"]))
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+        self.assertEqual(il_loaded, dict(title="foo"))
+        self.assertEqual(
+            NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo")
+        )
 
     def test_avoid_reprocessing_without_initial_values_single(self):
         il = NoInputReprocessingDictLoader()
-        il.add_value('title', 'foo')
+        il.add_value("title", "foo")
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+        self.assertEqual(il_loaded, dict(title="FOO"))
+        self.assertEqual(
+            NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO")
+        )
 
     def test_avoid_reprocessing_without_initial_values_list(self):
         il = NoInputReprocessingDictLoader()
-        il.add_value('title', ['foo', 'bar'])
+        il.add_value("title", ["foo", "bar"])
         il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+        self.assertEqual(il_loaded, dict(title="FOO"))
+        self.assertEqual(
+            NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO")
+        )
 
 
 class TestOutputProcessorDict(unittest.TestCase):
     def test_output_processor(self):
-
         class TempDict(dict):
             def __init__(self, *args, **kwargs):
                 super().__init__(self, *args, **kwargs)
-                self.setdefault('temp', 0.3)
+                self.setdefault("temp", 0.3)
 
         class TempLoader(ItemLoader):
             default_item_class = TempDict
@@ -589,64 +619,67 @@ class TestOutputProcessorDict(unittest.TestCase):
         loader = TempLoader()
         item = loader.load_item()
         self.assertIsInstance(item, TempDict)
-        self.assertEqual(dict(item), {'temp': 0.3})
+        self.assertEqual(dict(item), {"temp": 0.3})
 
 
 class ProcessorsTest(unittest.TestCase):
-
     def test_take_first(self):
         proc = TakeFirst()
-        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
-        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
+        self.assertEqual(proc([None, "", "hello", "world"]), "hello")
+        self.assertEqual(proc([None, "", 0, "hello", "world"]), 0)
 
     def test_identity(self):
         proc = Identity()
-        self.assertEqual(proc([None, '', 'hello', 'world']),
-                         [None, '', 'hello', 'world'])
+        self.assertEqual(
+            proc([None, "", "hello", "world"]), [None, "", "hello", "world"]
+        )
 
     def test_join(self):
         proc = Join()
-        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
-        self.assertEqual(proc(['hello', 'world']), 'hello world')
-        self.assertIsInstance(proc(['hello', 'world']), str)
+        self.assertRaises(TypeError, proc, [None, "", "hello", "world"])
+        self.assertEqual(proc(["", "hello", "world"]), " hello world")
+        self.assertEqual(proc(["hello", "world"]), "hello world")
+        self.assertIsInstance(proc(["hello", "world"]), str)
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
-        self.assertEqual(proc(['hello', 'world']), 'HELLO')
+        self.assertEqual(proc(["hello", "world"]), "HELLO")
         proc = Compose(str.upper)
         self.assertEqual(proc(None), None)
         proc = Compose(str.upper, stop_on_none=False)
         self.assertRaises(ValueError, proc, None)
         proc = Compose(str.upper, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
+        self.assertRaises(ValueError, proc, "hello")
 
     def test_mapcompose(self):
         def filter_world(x):
-            return None if x == 'world' else x
+            return None if x == "world" else x
+
         proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
-                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+        self.assertEqual(
+            proc(["hello", "world", "this", "is", "scrapy"]),
+            ["HELLO", "THIS", "IS", "SCRAPY"],
+        )
         proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc(None), [])
         proc = MapCompose(filter_world, str.upper)
         self.assertRaises(ValueError, proc, [1])
         proc = MapCompose(filter_world, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
+        self.assertRaises(ValueError, proc, "hello")
 
 
 class SelectJmesTestCase(unittest.TestCase):
     test_list_equals = {
-        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'dict': (
-            'foo.bar[*].name',
+        "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"),
+        "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None),
+        "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+        "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"),
+        "dict": (
+            "foo.bar[*].name",
             {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-            ['one', 'two']
+            ["one", "two"],
         ),
-        'list': ('[1]', [1, 2], 2)
+        "list": ("[1]", [1, 2], 2),
     }
 
     def test_output(self):
@@ -654,14 +687,13 @@ class SelectJmesTestCase(unittest.TestCase):
             expr, test_list, expected = self.test_list_equals[tl]
             test = SelectJmes(expr)(test_list)
             self.assertEqual(
-                test,
-                expected,
-                msg=f'test "{tl}" got {test} expected {expected}'
+                test, expected, msg=f'test "{tl}" got {test} expected {expected}'
             )
 
 
 # Functions as processors
 
+
 def function_processor_strip(iterable):
     return [x.strip() for x in iterable]
 
@@ -684,19 +716,14 @@ class FunctionProcessorDictLoader(ItemLoader):
 
 
 class FunctionProcessorTestCase(unittest.TestCase):
-
     def test_processor_defined_in_item_loader(self):
         lo = FunctionProcessorDictLoader()
-        lo.add_value('foo', '  bar  ')
-        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
-        self.assertEqual(
-            dict(lo.load_item()),
-            {'foo': ['BAR', 'ASDF', 'QWERTY']}
-        )
+        lo.add_value("foo", "  bar  ")
+        lo.add_value("foo", ["  asdf  ", "  qwerty  "])
+        self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
 
 
 class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
-
     def test_deprecated_wrap_loader_context(self):
         def function(*args):
             return None
@@ -709,7 +736,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
 
     def test_deprecated_extract_regex(self):
         with warnings.catch_warnings(record=True) as w:
-            extract_regex(r'\w+', 'this is a test')
+            extract_regex(r"\w+", "this is a test")
 
             assert len(w) == 1
             assert issubclass(w[0].category, ScrapyDeprecationWarning)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index f3bb23bda..11cf6d81a 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -24,67 +24,74 @@ class CustomItem(Item):
 
 
 class LogFormatterTestCase(unittest.TestCase):
-
     def setUp(self):
         self.formatter = LogFormatter()
-        self.spider = Spider('default')
+        self.spider = Spider("default")
 
     def test_crawled_with_referer(self):
         req = Request("http://www.example.com")
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
+        logline = logkws["msg"] % logkws["args"]
+        self.assertEqual(
+            logline, "Crawled (200) <GET http://www.example.com> (referer: None)"
+        )
 
     def test_crawled_without_referer(self):
-        req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
-        res = Response("http://www.example.com", flags=['cached'])
+        req = Request(
+            "http://www.example.com", headers={"referer": "http://example.com"}
+        )
+        res = Response("http://www.example.com", flags=["cached"])
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(
             logline,
-            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']",
+        )
 
     def test_flags_in_request(self):
-        req = Request("http://www.example.com", flags=['test', 'flag'])
+        req = Request("http://www.example.com", flags=["test", "flag"])
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(
             logline,
-            "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
+            "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)",
+        )
 
     def test_dropped(self):
         item = {}
         exception = Exception("\u2018")
         response = Response("http://www.example.com")
         logkws = self.formatter.dropped(item, exception, response, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, ["Dropped: \u2018", '{}'])
+        self.assertEqual(lines, ["Dropped: \u2018", "{}"])
 
     def test_item_error(self):
         # In practice, the complete traceback is shown by passing the
         # 'exc_info' argument to the logging function
-        item = {'key': 'value'}
+        item = {"key": "value"}
         exception = Exception()
         response = Response("http://www.example.com")
         logkws = self.formatter.item_error(item, exception, response, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(logline, "Error processing {'key': 'value'}")
 
     def test_spider_error(self):
         # In practice, the complete traceback is shown by passing the
         # 'exc_info' argument to the logging function
         failure = Failure(Exception())
-        request = Request("http://www.example.com", headers={'Referer': 'http://example.org'})
+        request = Request(
+            "http://www.example.com", headers={"Referer": "http://example.org"}
+        )
         response = Response("http://www.example.com", request=request)
         logkws = self.formatter.spider_error(failure, request, response, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(
             logline,
-            "Spider error processing <GET http://www.example.com> (referer: http://example.org)"
+            "Spider error processing <GET http://www.example.com> (referer: http://example.org)",
         )
 
     def test_download_error_short(self):
@@ -93,7 +100,7 @@ class LogFormatterTestCase(unittest.TestCase):
         failure = Failure(Exception())
         request = Request("http://www.example.com")
         logkws = self.formatter.download_error(failure, request, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(logline, "Error downloading <GET http://www.example.com>")
 
     def test_download_error_long(self):
@@ -101,67 +108,77 @@ class LogFormatterTestCase(unittest.TestCase):
         # 'exc_info' argument to the logging function
         failure = Failure(Exception())
         request = Request("http://www.example.com")
-        logkws = self.formatter.download_error(failure, request, self.spider, "Some message")
-        logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, "Error downloading <GET http://www.example.com>: Some message")
+        logkws = self.formatter.download_error(
+            failure, request, self.spider, "Some message"
+        )
+        logline = logkws["msg"] % logkws["args"]
+        self.assertEqual(
+            logline, "Error downloading <GET http://www.example.com>: Some message"
+        )
 
     def test_scraped(self):
         item = CustomItem()
-        item['name'] = '\xa3'
+        item["name"] = "\xa3"
         response = Response("http://www.example.com")
         logkws = self.formatter.scraped(item, response, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
+        self.assertEqual(
+            lines, ["Scraped from <200 http://www.example.com>", "name: \xa3"]
+        )
 
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
         kwargs = super().crawled(request, response, spider)
-        CRAWLEDMSG = (
-            "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
-        )
-        log_args = kwargs['args']
-        log_args['flags'] = str(request.flags)
+        CRAWLEDMSG = "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
+        log_args = kwargs["args"]
+        log_args["flags"] = str(request.flags)
         return {
-            'level': kwargs['level'],
-            'msg': CRAWLEDMSG,
-            'args': log_args,
+            "level": kwargs["level"],
+            "msg": CRAWLEDMSG,
+            "args": log_args,
         }
 
 
 class LogformatterSubclassTest(LogFormatterTestCase):
     def setUp(self):
         self.formatter = LogFormatterSubclass()
-        self.spider = Spider('default')
+        self.spider = Spider("default")
 
     def test_crawled_with_referer(self):
         req = Request("http://www.example.com")
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(
-            logline,
-            "Crawled (200) <GET http://www.example.com> (referer: None) []")
+            logline, "Crawled (200) <GET http://www.example.com> (referer: None) []"
+        )
 
     def test_crawled_without_referer(self):
-        req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached'])
+        req = Request(
+            "http://www.example.com",
+            headers={"referer": "http://example.com"},
+            flags=["cached"],
+        )
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(
             logline,
-            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']",
+        )
 
     def test_flags_in_request(self):
-        req = Request("http://www.example.com", flags=['test', 'flag'])
+        req = Request("http://www.example.com", flags=["test", "flag"])
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['msg'] % logkws['args']
+        logline = logkws["msg"] % logkws["args"]
         self.assertEqual(
             logline,
-            "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
+            "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']",
+        )
 
 
 class SkipMessagesLogFormatter(LogFormatter):
@@ -191,8 +208,8 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
         self.base_settings = {
-            'LOG_LEVEL': 'DEBUG',
-            'ITEM_PIPELINES': {
+            "LOG_LEVEL": "DEBUG",
+            "ITEM_PIPELINES": {
                 DropSomeItemsPipeline: 300,
             },
         }
@@ -212,7 +229,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
     @defer.inlineCallbacks
     def test_skip_messages(self):
         settings = self.base_settings.copy()
-        settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
+        settings["LOG_FORMATTER"] = SkipMessagesLogFormatter
         crawler = get_crawler(ItemSpider, settings)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 9b248fbfa..c78980d57 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -8,118 +8,144 @@ from scrapy.mail import MailSender
 
 
 class MailSenderTest(unittest.TestCase):
-
     def test_send(self):
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
-                        _callback=self._catch_mail_sent)
+        mailsender.send(
+            to=["test@scrapy.org"],
+            subject="subject",
+            body="body",
+            _callback=self._catch_mail_sent,
+        )
 
         assert self.catched_msg
 
-        self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
-        self.assertEqual(self.catched_msg['subject'], 'subject')
-        self.assertEqual(self.catched_msg['body'], 'body')
+        self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"])
+        self.assertEqual(self.catched_msg["subject"], "subject")
+        self.assertEqual(self.catched_msg["body"], "body")
 
-        msg = self.catched_msg['msg']
-        self.assertEqual(msg['to'], 'test@scrapy.org')
-        self.assertEqual(msg['subject'], 'subject')
-        self.assertEqual(msg.get_payload(), 'body')
-        self.assertEqual(msg.get('Content-Type'), 'text/plain')
+        msg = self.catched_msg["msg"]
+        self.assertEqual(msg["to"], "test@scrapy.org")
+        self.assertEqual(msg["subject"], "subject")
+        self.assertEqual(msg.get_payload(), "body")
+        self.assertEqual(msg.get("Content-Type"), "text/plain")
 
     def test_send_single_values_to_and_cc(self):
         mailsender = MailSender(debug=True)
-        mailsender.send(to='test@scrapy.org', subject='subject', body='body',
-                        cc='test@scrapy.org', _callback=self._catch_mail_sent)
+        mailsender.send(
+            to="test@scrapy.org",
+            subject="subject",
+            body="body",
+            cc="test@scrapy.org",
+            _callback=self._catch_mail_sent,
+        )
 
     def test_send_html(self):
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject='subject',
-                        body='<p>body</p>', mimetype='text/html',
-                        _callback=self._catch_mail_sent)
+        mailsender.send(
+            to=["test@scrapy.org"],
+            subject="subject",
+            body="<p>body</p>",
+            mimetype="text/html",
+            _callback=self._catch_mail_sent,
+        )
 
-        msg = self.catched_msg['msg']
-        self.assertEqual(msg.get_payload(), '<p>body</p>')
-        self.assertEqual(msg.get('Content-Type'), 'text/html')
+        msg = self.catched_msg["msg"]
+        self.assertEqual(msg.get_payload(), "<p>body</p>")
+        self.assertEqual(msg.get("Content-Type"), "text/html")
 
     def test_send_attach(self):
         attach = BytesIO()
-        attach.write(b'content')
+        attach.write(b"content")
         attach.seek(0)
-        attachs = [('attachment', 'text/plain', attach)]
+        attachs = [("attachment", "text/plain", attach)]
 
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
-                        attachs=attachs, _callback=self._catch_mail_sent)
+        mailsender.send(
+            to=["test@scrapy.org"],
+            subject="subject",
+            body="body",
+            attachs=attachs,
+            _callback=self._catch_mail_sent,
+        )
 
         assert self.catched_msg
-        self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
-        self.assertEqual(self.catched_msg['subject'], 'subject')
-        self.assertEqual(self.catched_msg['body'], 'body')
+        self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"])
+        self.assertEqual(self.catched_msg["subject"], "subject")
+        self.assertEqual(self.catched_msg["body"], "body")
 
-        msg = self.catched_msg['msg']
-        self.assertEqual(msg['to'], 'test@scrapy.org')
-        self.assertEqual(msg['subject'], 'subject')
+        msg = self.catched_msg["msg"]
+        self.assertEqual(msg["to"], "test@scrapy.org")
+        self.assertEqual(msg["subject"], "subject")
 
         payload = msg.get_payload()
         assert isinstance(payload, list)
         self.assertEqual(len(payload), 2)
 
         text, attach = payload
-        self.assertEqual(text.get_payload(decode=True), b'body')
-        self.assertEqual(text.get_charset(), Charset('us-ascii'))
-        self.assertEqual(attach.get_payload(decode=True), b'content')
+        self.assertEqual(text.get_payload(decode=True), b"body")
+        self.assertEqual(text.get_charset(), Charset("us-ascii"))
+        self.assertEqual(attach.get_payload(decode=True), b"content")
 
     def _catch_mail_sent(self, **kwargs):
         self.catched_msg = dict(**kwargs)
 
     def test_send_utf8(self):
-        subject = 'sübjèçt'
-        body = 'bödÿ-àéïöñß'
+        subject = "sübjèçt"
+        body = "bödÿ-àéïöñß"
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
-                        charset='utf-8', _callback=self._catch_mail_sent)
+        mailsender.send(
+            to=["test@scrapy.org"],
+            subject=subject,
+            body=body,
+            charset="utf-8",
+            _callback=self._catch_mail_sent,
+        )
 
         assert self.catched_msg
-        self.assertEqual(self.catched_msg['subject'], subject)
-        self.assertEqual(self.catched_msg['body'], body)
+        self.assertEqual(self.catched_msg["subject"], subject)
+        self.assertEqual(self.catched_msg["body"], body)
 
-        msg = self.catched_msg['msg']
-        self.assertEqual(msg['subject'], subject)
+        msg = self.catched_msg["msg"]
+        self.assertEqual(msg["subject"], subject)
         self.assertEqual(msg.get_payload(), body)
-        self.assertEqual(msg.get_charset(), Charset('utf-8'))
-        self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
+        self.assertEqual(msg.get_charset(), Charset("utf-8"))
+        self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"')
 
     def test_send_attach_utf8(self):
-        subject = 'sübjèçt'
-        body = 'bödÿ-àéïöñß'
+        subject = "sübjèçt"
+        body = "bödÿ-àéïöñß"
         attach = BytesIO()
-        attach.write(body.encode('utf-8'))
+        attach.write(body.encode("utf-8"))
         attach.seek(0)
-        attachs = [('attachment', 'text/plain', attach)]
+        attachs = [("attachment", "text/plain", attach)]
 
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
-                        attachs=attachs, charset='utf-8',
-                        _callback=self._catch_mail_sent)
+        mailsender.send(
+            to=["test@scrapy.org"],
+            subject=subject,
+            body=body,
+            attachs=attachs,
+            charset="utf-8",
+            _callback=self._catch_mail_sent,
+        )
 
         assert self.catched_msg
-        self.assertEqual(self.catched_msg['subject'], subject)
-        self.assertEqual(self.catched_msg['body'], body)
+        self.assertEqual(self.catched_msg["subject"], subject)
+        self.assertEqual(self.catched_msg["body"], body)
 
-        msg = self.catched_msg['msg']
-        self.assertEqual(msg['subject'], subject)
-        self.assertEqual(msg.get_charset(), Charset('utf-8'))
-        self.assertEqual(msg.get('Content-Type'),
-                         'multipart/mixed; charset="utf-8"')
+        msg = self.catched_msg["msg"]
+        self.assertEqual(msg["subject"], subject)
+        self.assertEqual(msg.get_charset(), Charset("utf-8"))
+        self.assertEqual(msg.get("Content-Type"), 'multipart/mixed; charset="utf-8"')
 
         payload = msg.get_payload()
         assert isinstance(payload, list)
         self.assertEqual(len(payload), 2)
 
         text, attach = payload
-        self.assertEqual(text.get_payload(decode=True).decode('utf-8'), body)
-        self.assertEqual(text.get_charset(), Charset('utf-8'))
-        self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
+        self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body)
+        self.assertEqual(text.get_charset(), Charset("utf-8"))
+        self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 8651431b5..a84cf4c28 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -6,7 +6,6 @@ from scrapy.middleware import MiddlewareManager
 
 
 class M1:
-
     def open_spider(self, spider):
         pass
 
@@ -18,7 +17,6 @@ class M1:
 
 
 class M2:
-
     def open_spider(self, spider):
         pass
 
@@ -29,13 +27,11 @@ class M2:
 
 
 class M3:
-
     def process(self, response, request, spider):
         pass
 
 
 class MOff:
-
     def open_spider(self, spider):
         pass
 
@@ -47,37 +43,39 @@ class MOff:
 
 
 class TestMiddlewareManager(MiddlewareManager):
-
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return [M1, MOff, M3]
 
     def _add_middleware(self, mw):
         super()._add_middleware(mw)
-        if hasattr(mw, 'process'):
-            self.methods['process'].append(mw.process)
+        if hasattr(mw, "process"):
+            self.methods["process"].append(mw.process)
 
 
 class MiddlewareManagerTest(unittest.TestCase):
-
     def test_init(self):
         m1, m2, m3 = M1(), M2(), M3()
         mwman = TestMiddlewareManager(m1, m2, m3)
-        self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider])
-        self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider])
-        self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process])
+        self.assertEqual(
+            list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider]
+        )
+        self.assertEqual(
+            list(mwman.methods["close_spider"]), [m2.close_spider, m1.close_spider]
+        )
+        self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process])
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
         self.assertEqual(
-            [x.__self__.__class__ for x in mwman.methods['open_spider']],
-            [M1, M2])
+            [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2]
+        )
         self.assertEqual(
-            [x.__self__.__class__ for x in mwman.methods['close_spider']],
-            [M2, M1])
+            [x.__self__.__class__ for x in mwman.methods["close_spider"]], [M2, M1]
+        )
         self.assertEqual(
-            [x.__self__.__class__ for x in mwman.methods['process']],
-            [M1, M3])
+            [x.__self__.__class__ for x in mwman.methods["process"]], [M1, M3]
+        )
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 39bca7bf2..b04da22be 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -13,7 +13,7 @@ from tests.spiders import SimpleSpider
 
 
 class MediaDownloadSpider(SimpleSpider):
-    name = 'mediadownload'
+    name = "mediadownload"
 
     def _process_url(self, url):
         return url
@@ -27,34 +27,38 @@ class MediaDownloadSpider(SimpleSpider):
                 self._process_url(response.urljoin(href))
                 for href in response.xpath(
                     '//table[thead/tr/th="Filename"]/tbody//a/@href'
-                ).getall()],
+                ).getall()
+            ],
         }
         yield item
 
 
 class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):
-    name = 'brokenmedia'
+    name = "brokenmedia"
 
     def _process_url(self, url):
-        return url + '.foo'
+        return url + ".foo"
 
 
 class RedirectedMediaDownloadSpider(MediaDownloadSpider):
-    name = 'redirectedmedia'
+    name = "redirectedmedia"
 
     def _process_url(self, url):
-        return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
+        return add_or_replace_parameter(
+            self.mockserver.url("/redirect-to"), "goto", url
+        )
 
 
 class FileDownloadCrawlTestCase(TestCase):
-    pipeline_class = 'scrapy.pipelines.files.FilesPipeline'
-    store_setting_key = 'FILES_STORE'
-    media_key = 'files'
-    media_urls_key = 'file_urls'
+    pipeline_class = "scrapy.pipelines.files.FilesPipeline"
+    store_setting_key = "FILES_STORE"
+    media_key = "files"
+    media_urls_key = "file_urls"
     expected_checksums = {
-        '5547178b89448faf0015a13f904c936e',
-        'c2281c83670e31d8aaab7cb642b824db',
-        'ed3f6538dc15d4d9179dae57319edc5f'}
+        "5547178b89448faf0015a13f904c936e",
+        "c2281c83670e31d8aaab7cb642b824db",
+        "ed3f6538dc15d4d9179dae57319edc5f",
+    }
 
     def setUp(self):
         self.mockserver = MockServer()
@@ -64,8 +68,8 @@ class FileDownloadCrawlTestCase(TestCase):
         self.tmpmediastore = Path(self.mktemp())
         self.tmpmediastore.mkdir()
         self.settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
-            'ITEM_PIPELINES': {self.pipeline_class: 1},
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+            "ITEM_PIPELINES": {self.pipeline_class: 1},
             self.store_setting_key: str(self.tmpmediastore),
         }
         self.runner = CrawlerRunner(self.settings)
@@ -91,27 +95,25 @@ class FileDownloadCrawlTestCase(TestCase):
         self.assertIn(self.media_key, items[0])
 
         # check that logs show the expected number of successful file downloads
-        file_dl_success = 'File (downloaded): Downloaded file from'
+        file_dl_success = "File (downloaded): Downloaded file from"
         self.assertEqual(logs.count(file_dl_success), 3)
 
         # check that the images/files status is `downloaded`
         for item in items:
             for i in item[self.media_key]:
-                self.assertEqual(i['status'], 'downloaded')
+                self.assertEqual(i["status"], "downloaded")
 
         # check that the images/files checksums are what we know they should be
         if self.expected_checksums is not None:
             checksums = set(
-                i['checksum']
-                for item in items
-                for i in item[self.media_key]
+                i["checksum"] for item in items for i in item[self.media_key]
             )
             self.assertEqual(checksums, self.expected_checksums)
 
         # check that the image files where actually written to the media store
         for item in items:
             for i in item[self.media_key]:
-                self.assertTrue((self.tmpmediastore / i['path']).exists())
+                self.assertTrue((self.tmpmediastore / i["path"]).exists())
 
     def _assert_files_download_failure(self, crawler, items, code, logs):
 
@@ -121,13 +123,19 @@ class FileDownloadCrawlTestCase(TestCase):
         self.assertFalse(items[0][self.media_key])
 
         # check that there was 1 successful fetch and 3 other responses with non-200 code
-        self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4)
-        self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4)
-        self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1)
-        self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3)
+        self.assertEqual(
+            crawler.stats.get_value("downloader/request_method_count/GET"), 4
+        )
+        self.assertEqual(crawler.stats.get_value("downloader/response_count"), 4)
+        self.assertEqual(
+            crawler.stats.get_value("downloader/response_status_count/200"), 1
+        )
+        self.assertEqual(
+            crawler.stats.get_value(f"downloader/response_status_count/{code}"), 3
+        )
 
         # check that logs do show the failure on the file downloads
-        file_dl_failure = f'File (code: {code}): Error downloading file from'
+        file_dl_failure = f"File (code: {code}): Error downloading file from"
         self.assertEqual(logs.count(file_dl_failure), 3)
 
         # check that no files were written to the media store
@@ -140,7 +148,8 @@ class FileDownloadCrawlTestCase(TestCase):
             yield crawler.crawl(
                 self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
-                media_urls_key=self.media_urls_key)
+                media_urls_key=self.media_urls_key,
+            )
         self._assert_files_downloaded(self.items, str(log))
 
     @defer.inlineCallbacks
@@ -150,7 +159,8 @@ class FileDownloadCrawlTestCase(TestCase):
             yield crawler.crawl(
                 self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
-                media_urls_key=self.media_urls_key)
+                media_urls_key=self.media_urls_key,
+            )
         self._assert_files_download_failure(crawler, self.items, 404, str(log))
 
     @defer.inlineCallbacks
@@ -161,13 +171,14 @@ class FileDownloadCrawlTestCase(TestCase):
                 self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
-                mockserver=self.mockserver)
+                mockserver=self.mockserver,
+            )
         self._assert_files_download_failure(crawler, self.items, 302, str(log))
 
     @defer.inlineCallbacks
     def test_download_media_redirected_allowed(self):
         settings = dict(self.settings)
-        settings.update({'MEDIA_ALLOW_REDIRECTS': True})
+        settings.update({"MEDIA_ALLOW_REDIRECTS": True})
         runner = CrawlerRunner(settings)
         crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
         with LogCapture() as log:
@@ -175,15 +186,20 @@ class FileDownloadCrawlTestCase(TestCase):
                 self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
-                mockserver=self.mockserver)
+                mockserver=self.mockserver,
+            )
         self._assert_files_downloaded(self.items, str(log))
-        self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
+        self.assertEqual(
+            crawler.stats.get_value("downloader/response_status_count/302"), 3
+        )
 
 
 try:
     from PIL import Image  # noqa: imported just to check for the import error
 except ImportError:
-    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+    skip_pillow = (
+        "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+    )
 else:
     skip_pillow = None
 
@@ -192,10 +208,10 @@ class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
 
     skip = skip_pillow
 
-    pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
-    store_setting_key = 'IMAGES_STORE'
-    media_key = 'images'
-    media_urls_key = 'image_urls'
+    pipeline_class = "scrapy.pipelines.images.ImagesPipeline"
+    store_setting_key = "IMAGES_STORE"
+    media_key = "images"
+    media_urls_key = "image_urls"
 
     # somehow checksums for images are different for Python 3.3
     expected_checksums = None
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4acd29bf7..43942e53e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -35,15 +35,14 @@ from scrapy.utils.test import (
 
 
 def _mocked_download_func(request, info):
-    response = request.meta.get('response')
+    response = request.meta.get("response")
     return response() if callable(response) else response
 
 
 class FilesPipelineTestCase(unittest.TestCase):
-
     def setUp(self):
         self.tempdir = mkdtemp()
-        settings_dict = {'FILES_STORE': self.tempdir}
+        settings_dict = {"FILES_STORE": self.tempdir}
         crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
         self.pipeline = FilesPipeline.from_crawler(crawler)
         self.pipeline.download_func = _mocked_download_func
@@ -56,41 +55,70 @@ class FilesPipelineTestCase(unittest.TestCase):
         file_path = self.pipeline.file_path
         self.assertEqual(
             file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
-            'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
+            "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf",
+        )
         self.assertEqual(
-            file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
-            'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
+            file_path(
+                Request(
+                    "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"
+                )
+            ),
+            "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt",
+        )
         self.assertEqual(
-            file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
-            'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
+            file_path(
+                Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")
+            ),
+            "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc",
+        )
         self.assertEqual(
-            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
-            'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+            file_path(
+                Request(
+                    "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"
+                )
+            ),
+            "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg",
+        )
         self.assertEqual(
             file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
-            'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
+            "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2",
+        )
         self.assertEqual(
             file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
-            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+            "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1",
+        )
         self.assertEqual(
-            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
-                      response=Response("http://www.dorma.co.uk/images/product_details/2532"),
-                      info=object()),
-            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+            file_path(
+                Request("http://www.dorma.co.uk/images/product_details/2532"),
+                response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+                info=object(),
+            ),
+            "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1",
+        )
         self.assertEqual(
-            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
-            'full/76c00cef2ef669ae65052661f68d451162829507')
+            file_path(
+                Request(
+                    "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha"
+                )
+            ),
+            "full/76c00cef2ef669ae65052661f68d451162829507",
+        )
         self.assertEqual(
-            file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
-                                    //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
-            'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+            file_path(
+                Request(
+                    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+                                    //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y"
+                )
+            ),
+            "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png",
+        )
 
     def test_fs_store(self):
         assert isinstance(self.pipeline.store, FSFilesStore)
         self.assertEqual(self.pipeline.store.basedir, self.tempdir)
 
-        path = 'some/image/key.jpg'
-        fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg')
+        path = "some/image/key.jpg"
+        fullpath = Path(self.tempdir, "some", "image", "key.jpg")
         self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath)
 
     @defer.inlineCallbacks
@@ -98,18 +126,24 @@ class FilesPipelineTestCase(unittest.TestCase):
         item_url = "http://example.com/file.pdf"
         item = _create_item_with_files(item_url)
         patchers = [
-            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
-            mock.patch.object(FSFilesStore, 'stat_file', return_value={
-                'checksum': 'abc', 'last_modified': time.time()}),
-            mock.patch.object(FilesPipeline, 'get_media_requests',
-                              return_value=[_prepare_request_object(item_url)])
+            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+            mock.patch.object(
+                FSFilesStore,
+                "stat_file",
+                return_value={"checksum": "abc", "last_modified": time.time()},
+            ),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
         ]
         for p in patchers:
             p.start()
 
         result = yield self.pipeline.process_item(item, None)
-        self.assertEqual(result['files'][0]['checksum'], 'abc')
-        self.assertEqual(result['files'][0]['status'], 'uptodate')
+        self.assertEqual(result["files"][0]["checksum"], "abc")
+        self.assertEqual(result["files"][0]["status"], "uptodate")
 
         for p in patchers:
             p.stop()
@@ -119,19 +153,28 @@ class FilesPipelineTestCase(unittest.TestCase):
         item_url = "http://example.com/file2.pdf"
         item = _create_item_with_files(item_url)
         patchers = [
-            mock.patch.object(FSFilesStore, 'stat_file', return_value={
-                'checksum': 'abc',
-                'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
-            mock.patch.object(FilesPipeline, 'get_media_requests',
-                              return_value=[_prepare_request_object(item_url)]),
-            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)
+            mock.patch.object(
+                FSFilesStore,
+                "stat_file",
+                return_value={
+                    "checksum": "abc",
+                    "last_modified": time.time()
+                    - (self.pipeline.expires * 60 * 60 * 24 * 2),
+                },
+            ),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
         ]
         for p in patchers:
             p.start()
 
         result = yield self.pipeline.process_item(item, None)
-        self.assertNotEqual(result['files'][0]['checksum'], 'abc')
-        self.assertEqual(result['files'][0]['status'], 'downloaded')
+        self.assertNotEqual(result["files"][0]["checksum"], "abc")
+        self.assertEqual(result["files"][0]["status"], "downloaded")
 
         for p in patchers:
             p.stop()
@@ -141,19 +184,28 @@ class FilesPipelineTestCase(unittest.TestCase):
         item_url = "http://example.com/file3.pdf"
         item = _create_item_with_files(item_url)
         patchers = [
-            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
-            mock.patch.object(FSFilesStore, 'stat_file', return_value={
-                'checksum': 'abc',
-                'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
-            mock.patch.object(FilesPipeline, 'get_media_requests',
-                              return_value=[_prepare_request_object(item_url, flags=['cached'])])
+            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+            mock.patch.object(
+                FSFilesStore,
+                "stat_file",
+                return_value={
+                    "checksum": "abc",
+                    "last_modified": time.time()
+                    - (self.pipeline.expires * 60 * 60 * 24 * 2),
+                },
+            ),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url, flags=["cached"])],
+            ),
         ]
         for p in patchers:
             p.start()
 
         result = yield self.pipeline.process_item(item, None)
-        self.assertNotEqual(result['files'][0]['checksum'], 'abc')
-        self.assertEqual(result['files'][0]['status'], 'cached')
+        self.assertNotEqual(result["files"][0]["checksum"], "abc")
+        self.assertEqual(result["files"][0]["status"], "cached")
 
         for p in patchers:
             p.stop()
@@ -162,48 +214,58 @@ class FilesPipelineTestCase(unittest.TestCase):
         """
         Custom file path based on item data, overriding default implementation
         """
+
         class CustomFilesPipeline(FilesPipeline):
             def file_path(self, request, response=None, info=None, item=None):
                 return f'full/{item.get("path")}'
 
-        file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path
-        item = dict(path='path-to-store-file')
+        file_path = CustomFilesPipeline.from_settings(
+            Settings({"FILES_STORE": self.tempdir})
+        ).file_path
+        item = dict(path="path-to-store-file")
         request = Request("http://example.com")
-        self.assertEqual(file_path(request, item=item), 'full/path-to-store-file')
+        self.assertEqual(file_path(request, item=item), "full/path-to-store-file")
 
 
 class FilesPipelineTestCaseFieldsMixin:
-
     def test_item_fields_default(self):
-        url = 'http://www.example.com/files/1.txt'
-        item = self.item_class(name='item1', file_urls=[url])
-        pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
+        url = "http://www.example.com/files/1.txt"
+        item = self.item_class(name="item1", file_urls=[url])
+        pipeline = FilesPipeline.from_settings(
+            Settings({"FILES_STORE": "s3://example/files/"})
+        )
         requests = list(pipeline.get_media_requests(item, None))
         self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
+        results = [(True, {"url": url})]
         item = pipeline.item_completed(results, item, None)
         files = ItemAdapter(item).get("files")
         self.assertEqual(files, [results[0][1]])
         self.assertIsInstance(item, self.item_class)
 
     def test_item_fields_override_settings(self):
-        url = 'http://www.example.com/files/1.txt'
-        item = self.item_class(name='item1', custom_file_urls=[url])
-        pipeline = FilesPipeline.from_settings(Settings({
-            'FILES_STORE': 's3://example/files/',
-            'FILES_URLS_FIELD': 'custom_file_urls',
-            'FILES_RESULT_FIELD': 'custom_files'
-        }))
+        url = "http://www.example.com/files/1.txt"
+        item = self.item_class(name="item1", custom_file_urls=[url])
+        pipeline = FilesPipeline.from_settings(
+            Settings(
+                {
+                    "FILES_STORE": "s3://example/files/",
+                    "FILES_URLS_FIELD": "custom_file_urls",
+                    "FILES_RESULT_FIELD": "custom_files",
+                }
+            )
+        )
         requests = list(pipeline.get_media_requests(item, None))
         self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
+        results = [(True, {"url": url})]
         item = pipeline.item_completed(results, item, None)
         custom_files = ItemAdapter(item).get("custom_files")
         self.assertEqual(custom_files, [results[0][1]])
         self.assertIsInstance(item, self.item_class)
 
 
-class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsDict(
+    FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = dict
 
 
@@ -217,7 +279,9 @@ class FilesPipelineTestItem(Item):
     custom_files = Field()
 
 
-class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsItem(
+    FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = FilesPipelineTestItem
 
 
@@ -232,7 +296,9 @@ class FilesPipelineTestDataClass:
     custom_files: list = dataclasses.field(default_factory=list)
 
 
-class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsDataClass(
+    FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = FilesPipelineTestDataClass
 
 
@@ -247,7 +313,9 @@ class FilesPipelineTestAttrsItem:
     custom_files = attr.ib(default=lambda: [])
 
 
-class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsAttrsItem(
+    FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = FilesPipelineTestAttrsItem
 
 
@@ -255,12 +323,12 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
     default_cls_settings = {
         "EXPIRES": 90,
         "FILES_URLS_FIELD": "file_urls",
-        "FILES_RESULT_FIELD": "files"
+        "FILES_RESULT_FIELD": "files",
     }
     file_cls_attr_settings_map = {
         ("EXPIRES", "FILES_EXPIRES", "expires"),
         ("FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"),
-        ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field")
+        ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),
     }
 
     def setUp(self):
@@ -270,7 +338,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         rmtree(self.tempdir)
 
     def _generate_fake_settings(self, prefix=None):
-
         def random_string():
             return "".join([chr(random.randint(97, 123)) for _ in range(10)])
 
@@ -278,15 +345,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             "FILES_EXPIRES": random.randint(100, 1000),
             "FILES_URLS_FIELD": random_string(),
             "FILES_RESULT_FIELD": random_string(),
-            "FILES_STORE": self.tempdir
+            "FILES_STORE": self.tempdir,
         }
         if not prefix:
             return settings
 
-        return {prefix.upper() + "_" + k if k != "FILES_STORE" else k: v for k, v in settings.items()}
+        return {
+            prefix.upper() + "_" + k if k != "FILES_STORE" else k: v
+            for k, v in settings.items()
+        }
 
     def _generate_fake_pipeline(self):
-
         class UserDefinedFilePipeline(FilesPipeline):
             EXPIRES = 1001
             FILES_URLS_FIELD = "alfa"
@@ -339,10 +408,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         If there are no settings for subclass and no subclass attributes, pipeline should use
         attributes of base class.
         """
+
         class UserDefinedFilesPipeline(FilesPipeline):
             pass
 
-        user_pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
+        user_pipeline = UserDefinedFilesPipeline.from_settings(
+            Settings({"FILES_STORE": self.tempdir})
+        )
         for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
             # Values from settings for custom pipeline should be set on pipeline instance.
             custom_value = self.default_cls_settings.get(pipe_attr.upper())
@@ -353,6 +425,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         If there are custom settings for subclass and NO class attributes, pipeline should use custom
         settings.
         """
+
         class UserDefinedFilesPipeline(FilesPipeline):
             pass
 
@@ -374,7 +447,11 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         prefix = pipeline_cls.__name__.upper()
         settings = self._generate_fake_settings(prefix=prefix)
         user_pipeline = pipeline_cls.from_settings(Settings(settings))
-        for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
+        for (
+            pipe_cls_attr,
+            settings_attr,
+            pipe_inst_attr,
+        ) in self.file_cls_attr_settings_map:
             custom_value = settings.get(prefix + "_" + settings_attr)
             self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr])
             self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
@@ -384,7 +461,9 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             DEFAULT_FILES_RESULT_FIELD = "this"
             DEFAULT_FILES_URLS_FIELD = "that"
 
-        pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
+        pipeline = UserDefinedFilesPipeline.from_settings(
+            Settings({"FILES_STORE": self.tempdir})
+        )
         self.assertEqual(pipeline.files_result_field, "this")
         self.assertEqual(pipeline.files_urls_field, "that")
 
@@ -402,37 +481,36 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
 
         for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
             expected_value = settings.get(settings_attr)
-            self.assertEqual(getattr(pipeline_cls, pipe_inst_attr),
-                             expected_value)
+            self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value)
 
 
 class TestS3FilesStore(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_persist(self):
         skip_if_no_boto()
 
-        bucket = 'mybucket'
-        key = 'export.csv'
-        uri = f's3://{bucket}/{key}'
+        bucket = "mybucket"
+        key = "export.csv"
+        uri = f"s3://{bucket}/{key}"
         buffer = mock.MagicMock()
-        meta = {'foo': 'bar'}
-        path = ''
-        content_type = 'image/png'
+        meta = {"foo": "bar"}
+        path = ""
+        content_type = "image/png"
 
         store = S3FilesStore(uri)
         from botocore.stub import Stubber
+
         with Stubber(store.s3_client) as stub:
             stub.add_response(
-                'put_object',
+                "put_object",
                 expected_params={
-                    'ACL': S3FilesStore.POLICY,
-                    'Body': buffer,
-                    'Bucket': bucket,
-                    'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
-                    'ContentType': content_type,
-                    'Key': key,
-                    'Metadata': meta,
+                    "ACL": S3FilesStore.POLICY,
+                    "Body": buffer,
+                    "Bucket": bucket,
+                    "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+                    "ContentType": content_type,
+                    "Key": key,
+                    "Metadata": meta,
                 },
                 service_response={},
             )
@@ -442,7 +520,7 @@ class TestS3FilesStore(unittest.TestCase):
                 buffer,
                 info=None,
                 meta=meta,
-                headers={'Content-Type': content_type},
+                headers={"Content-Type": content_type},
             )
 
             stub.assert_no_pending_responses()
@@ -451,40 +529,41 @@ class TestS3FilesStore(unittest.TestCase):
                 [
                     mock.call.seek(0),
                     # The call to read does not happen with Stubber
-                ]
+                ],
             )
 
     @defer.inlineCallbacks
     def test_stat(self):
         skip_if_no_boto()
 
-        bucket = 'mybucket'
-        key = 'export.csv'
-        uri = f's3://{bucket}/{key}'
-        checksum = '3187896a9657a28163abb31667df64c8'
+        bucket = "mybucket"
+        key = "export.csv"
+        uri = f"s3://{bucket}/{key}"
+        checksum = "3187896a9657a28163abb31667df64c8"
         last_modified = datetime(2019, 12, 1)
 
         store = S3FilesStore(uri)
         from botocore.stub import Stubber
+
         with Stubber(store.s3_client) as stub:
             stub.add_response(
-                'head_object',
+                "head_object",
                 expected_params={
-                    'Bucket': bucket,
-                    'Key': key,
+                    "Bucket": bucket,
+                    "Key": key,
                 },
                 service_response={
-                    'ETag': f'"{checksum}"',
-                    'LastModified': last_modified,
+                    "ETag": f'"{checksum}"',
+                    "LastModified": last_modified,
                 },
             )
 
-            file_stats = yield store.stat_file('', info=None)
+            file_stats = yield store.stat_file("", info=None)
             self.assertEqual(
                 file_stats,
                 {
-                    'checksum': checksum,
-                    'last_modified': last_modified.timestamp(),
+                    "checksum": checksum,
+                    "last_modified": last_modified.timestamp(),
                 },
             )
 
@@ -495,27 +574,27 @@ class TestGCSFilesStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
         assert_gcs_environ()
-        uri = os.environ.get('GCS_TEST_FILE_URI')
+        uri = os.environ.get("GCS_TEST_FILE_URI")
         if not uri:
             raise unittest.SkipTest("No GCS URI available for testing")
         data = b"TestGCSFilesStore: \xe2\x98\x83"
         buf = BytesIO(data)
-        meta = {'foo': 'bar'}
-        path = 'full/filename'
+        meta = {"foo": "bar"}
+        path = "full/filename"
         store = GCSFilesStore(uri)
-        store.POLICY = 'authenticatedRead'
-        expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'}
+        store.POLICY = "authenticatedRead"
+        expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"}
         yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
         s = yield store.stat_file(path, info=None)
-        self.assertIn('last_modified', s)
-        self.assertIn('checksum', s)
-        self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
+        self.assertIn("last_modified", s)
+        self.assertIn("checksum", s)
+        self.assertEqual(s["checksum"], "zc2oVgXkbQr2EQdSdw3OPA==")
         u = urlparse(uri)
         content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path)
         self.assertEqual(content, data)
-        self.assertEqual(blob.metadata, {'foo': 'bar'})
+        self.assertEqual(blob.metadata, {"foo": "bar"})
         self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
-        self.assertEqual(blob.content_type, 'application/octet-stream')
+        self.assertEqual(blob.content_type, "application/octet-stream")
         self.assertIn(expected_policy, acl)
 
     @defer.inlineCallbacks
@@ -525,17 +604,19 @@ class TestGCSFilesStore(unittest.TestCase):
         """
         assert_gcs_environ()
         try:
-            import google.cloud.storage # noqa
+            import google.cloud.storage  # noqa
         except ModuleNotFoundError:
             raise unittest.SkipTest("google-cloud-storage is not installed")
         else:
-            with mock.patch('google.cloud.storage') as _:
-                with mock.patch('scrapy.pipelines.files.time') as _:
-                    uri = 'gs://my_bucket/my_prefix/'
+            with mock.patch("google.cloud.storage") as _:
+                with mock.patch("scrapy.pipelines.files.time") as _:
+                    uri = "gs://my_bucket/my_prefix/"
                     store = GCSFilesStore(uri)
                     store.bucket = mock.Mock()
-                    path = 'full/my_data.txt'
-                    yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None)
+                    path = "full/my_data.txt"
+                    yield store.persist_file(
+                        path, mock.Mock(), info=None, meta=None, headers=None
+                    )
                     yield store.stat_file(path, info=None)
                     expected_blob_path = store.prefix + path
                     store.bucket.blob.assert_called_with(expected_blob_path)
@@ -545,25 +626,30 @@ class TestGCSFilesStore(unittest.TestCase):
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
-        uri = os.environ.get('FTP_TEST_FILE_URI')
+        uri = os.environ.get("FTP_TEST_FILE_URI")
         if not uri:
             raise unittest.SkipTest("No FTP URI available for testing")
         data = b"TestFTPFilesStore: \xe2\x98\x83"
         buf = BytesIO(data)
-        meta = {'foo': 'bar'}
-        path = 'full/filename'
+        meta = {"foo": "bar"}
+        path = "full/filename"
         store = FTPFilesStore(uri)
         empty_dict = yield store.stat_file(path, info=None)
         self.assertEqual(empty_dict, {})
         yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
         stat = yield store.stat_file(path, info=None)
-        self.assertIn('last_modified', stat)
-        self.assertIn('checksum', stat)
-        self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')
-        path = f'{store.basedir}/{path}'
+        self.assertIn("last_modified", stat)
+        self.assertIn("checksum", stat)
+        self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6")
+        path = f"{store.basedir}/{path}"
         content = get_ftp_content_and_delete(
-            path, store.host, store.port,
-            store.username, store.password, store.USE_ACTIVE_MODE)
+            path,
+            store.host,
+            store.port,
+            store.username,
+            store.password,
+            store.USE_ACTIVE_MODE,
+        )
         self.assertEqual(data.decode(), content)
 
 
@@ -574,14 +660,15 @@ class ItemWithFiles(Item):
 
 def _create_item_with_files(*files):
     item = ItemWithFiles()
-    item['file_urls'] = files
+    item["file_urls"] = files
     return item
 
 
 def _prepare_request_object(item_url, flags=None):
     return Request(
         item_url,
-        meta={'response': Response(item_url, status=200, body=b'data', flags=flags)})
+        meta={"response": Response(item_url, status=200, body=b"data", flags=flags)},
+    )
 
 
 if __name__ == "__main__":
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f98d40fda..3cd3e5966 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -22,17 +22,19 @@ from scrapy.utils.python import to_bytes
 try:
     from PIL import Image
 except ImportError:
-    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+    skip_pillow = (
+        "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+    )
 else:
-    encoders = {'jpeg_encoder', 'jpeg_decoder'}
+    encoders = {"jpeg_encoder", "jpeg_decoder"}
     if not encoders.issubset(set(Image.core.__dict__)):
-        skip_pillow = 'Missing JPEG encoders'
+        skip_pillow = "Missing JPEG encoders"
     else:
         skip_pillow = None
 
 
 def _mocked_download_func(request, info):
-    response = request.meta.get('response')
+    response = request.meta.get("response")
     return response() if callable(response) else response
 
 
@@ -42,7 +44,9 @@ class ImagesPipelineTestCase(unittest.TestCase):
 
     def setUp(self):
         self.tempdir = mkdtemp()
-        self.pipeline = ImagesPipeline(self.tempdir, download_func=_mocked_download_func)
+        self.pipeline = ImagesPipeline(
+            self.tempdir, download_func=_mocked_download_func
+        )
 
     def tearDown(self):
         rmtree(self.tempdir)
@@ -51,43 +55,75 @@ class ImagesPipelineTestCase(unittest.TestCase):
         file_path = self.pipeline.file_path
         self.assertEqual(
             file_path(Request("https://dev.mydeco.com/mydeco.gif")),
-            'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+            "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
+        )
         self.assertEqual(
-            file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
-            'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
+            file_path(
+                Request(
+                    "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"
+                )
+            ),
+            "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg",
+        )
         self.assertEqual(
-            file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
-            'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
+            file_path(
+                Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")
+            ),
+            "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg",
+        )
         self.assertEqual(
-            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
-            'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+            file_path(
+                Request(
+                    "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"
+                )
+            ),
+            "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg",
+        )
         self.assertEqual(
             file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
-            'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
+            "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg",
+        )
         self.assertEqual(
             file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
-            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+            "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg",
+        )
         self.assertEqual(
-            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
-                      response=Response("http://www.dorma.co.uk/images/product_details/2532"),
-                      info=object()),
-            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+            file_path(
+                Request("http://www.dorma.co.uk/images/product_details/2532"),
+                response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+                info=object(),
+            ),
+            "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg",
+        )
 
     def test_thumbnail_name(self):
         thumb_path = self.pipeline.thumb_path
-        name = '50'
-        self.assertEqual(thumb_path(Request("file:///tmp/foo.jpg"), name),
-                         'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
-        self.assertEqual(thumb_path(Request("file://foo.png"), name),
-                         'thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg')
-        self.assertEqual(thumb_path(Request("file:///tmp/foo"), name),
-                         'thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg')
-        self.assertEqual(thumb_path(Request("file:///tmp/some.name/foo"), name),
-                         'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
-        self.assertEqual(thumb_path(Request("file:///tmp/some.name/foo"), name,
-                                    response=Response("file:///tmp/some.name/foo"),
-                                    info=object()),
-                         'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
+        name = "50"
+        self.assertEqual(
+            thumb_path(Request("file:///tmp/foo.jpg"), name),
+            "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg",
+        )
+        self.assertEqual(
+            thumb_path(Request("file://foo.png"), name),
+            "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg",
+        )
+        self.assertEqual(
+            thumb_path(Request("file:///tmp/foo"), name),
+            "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg",
+        )
+        self.assertEqual(
+            thumb_path(Request("file:///tmp/some.name/foo"), name),
+            "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg",
+        )
+        self.assertEqual(
+            thumb_path(
+                Request("file:///tmp/some.name/foo"),
+                name,
+                response=Response("file:///tmp/some.name/foo"),
+                info=object(),
+            ),
+            "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg",
+        )
 
     def test_thumbnail_name_from_item(self):
         """
@@ -95,23 +131,27 @@ class ImagesPipelineTestCase(unittest.TestCase):
         """
 
         class CustomImagesPipeline(ImagesPipeline):
-            def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+            def thumb_path(
+                self, request, thumb_id, response=None, info=None, item=None
+            ):
                 return f"thumb/{thumb_id}/{item.get('path')}"
 
-        thumb_path = CustomImagesPipeline.from_settings(Settings(
-            {'IMAGES_STORE': self.tempdir}
-        )).thumb_path
-        item = dict(path='path-to-store-file')
+        thumb_path = CustomImagesPipeline.from_settings(
+            Settings({"IMAGES_STORE": self.tempdir})
+        ).thumb_path
+        item = dict(path="path-to-store-file")
         request = Request("http://example.com")
-        self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
+        self.assertEqual(
+            thumb_path(request, "small", item=item), "thumb/small/path-to-store-file"
+        )
 
     def test_get_images_exception(self):
         self.pipeline.min_width = 100
         self.pipeline.min_height = 100
 
-        _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
-        _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0))
-        _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0))
+        _, buf1 = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+        _, buf2 = _create_image("JPEG", "RGB", (150, 50), (0, 0, 0))
+        _, buf3 = _create_image("JPEG", "RGB", (50, 150), (0, 0, 0))
 
         resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue())
         resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue())
@@ -128,121 +168,154 @@ class ImagesPipelineTestCase(unittest.TestCase):
     def test_get_images_new(self):
         self.pipeline.min_width = 0
         self.pipeline.min_height = 0
-        self.pipeline.thumbs = {'small': (20, 20)}
+        self.pipeline.thumbs = {"small": (20, 20)}
 
-        orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
-        orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0))
+        orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+        orig_thumb, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0))
         resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
         req = Request(url="https://dev.mydeco.com/mydeco.gif")
 
-        get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object())
+        get_images_gen = self.pipeline.get_images(
+            response=resp, request=req, info=object()
+        )
 
         path, new_im, new_buf = next(get_images_gen)
-        self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+        self.assertEqual(path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg")
         self.assertEqual(orig_im, new_im)
         self.assertEqual(buf.getvalue(), new_buf.getvalue())
 
         thumb_path, thumb_img, thumb_buf = next(get_images_gen)
-        self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+        self.assertEqual(
+            thumb_path, "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+        )
         self.assertEqual(thumb_img, thumb_img)
         self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())
 
     def test_get_images_old(self):
-        self.pipeline.thumbs = {'small': (20, 20)}
-        orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
+        self.pipeline.thumbs = {"small": (20, 20)}
+        orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
         resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
         req = Request(url="https://dev.mydeco.com/mydeco.gif")
 
         def overridden_convert_image(image, size=None):
-            im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
+            im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
             return im, buf
 
-        with patch.object(self.pipeline, 'convert_image', overridden_convert_image):
+        with patch.object(self.pipeline, "convert_image", overridden_convert_image):
             with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter('always')
-                get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object())
+                warnings.simplefilter("always")
+                get_images_gen = self.pipeline.get_images(
+                    response=resp, request=req, info=object()
+                )
                 path, new_im, new_buf = next(get_images_gen)
-                self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+                self.assertEqual(
+                    path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+                )
                 self.assertEqual(orig_im.mode, new_im.mode)
                 self.assertEqual(orig_im.getcolors(), new_im.getcolors())
                 self.assertEqual(buf.getvalue(), new_buf.getvalue())
 
                 thumb_path, thumb_img, thumb_buf = next(get_images_gen)
-                self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+                self.assertEqual(
+                    thumb_path,
+                    "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
+                )
                 self.assertEqual(orig_im.mode, thumb_img.mode)
                 self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
                 self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
 
-                expected_warning_msg = ('.convert_image() method overriden in a deprecated way, '
-                                        'overriden method does not accept response_body argument.')
-                self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1)
+                expected_warning_msg = (
+                    ".convert_image() method overriden in a deprecated way, "
+                    "overriden method does not accept response_body argument."
+                )
+                self.assertEqual(
+                    len(
+                        [
+                            warning
+                            for warning in w
+                            if expected_warning_msg in str(warning.message)
+                        ]
+                    ),
+                    1,
+                )
 
     def test_convert_image_old(self):
         # tests for old API
         with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
+            warnings.simplefilter("always")
             SIZE = (100, 100)
             # straigh forward case: RGB and JPEG
             COLOUR = (0, 127, 255)
-            im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR)
+            im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)
             converted, _ = self.pipeline.convert_image(im)
-            self.assertEqual(converted.mode, 'RGB')
+            self.assertEqual(converted.mode, "RGB")
             self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
 
             # check that thumbnail keep image ratio
             thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))
-            self.assertEqual(thumbnail.mode, 'RGB')
+            self.assertEqual(thumbnail.mode, "RGB")
             self.assertEqual(thumbnail.size, (10, 10))
 
             # transparency case: RGBA and PNG
             COLOUR = (0, 127, 255, 50)
-            im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR)
+            im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
             converted, _ = self.pipeline.convert_image(im)
-            self.assertEqual(converted.mode, 'RGB')
+            self.assertEqual(converted.mode, "RGB")
             self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
             # transparency case with palette: P and PNG
             COLOUR = (0, 127, 255, 50)
-            im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR)
-            im = im.convert('P')
+            im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
+            im = im.convert("P")
             converted, _ = self.pipeline.convert_image(im)
-            self.assertEqual(converted.mode, 'RGB')
+            self.assertEqual(converted.mode, "RGB")
             self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
             # ensure that we recieved deprecation warnings
-            expected_warning_msg = '.convert_image() method called in a deprecated way'
-            self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4)
+            expected_warning_msg = ".convert_image() method called in a deprecated way"
+            self.assertTrue(
+                len(
+                    [
+                        warning
+                        for warning in w
+                        if expected_warning_msg in str(warning.message)
+                    ]
+                )
+                == 4
+            )
 
     def test_convert_image_new(self):
         # tests for new API
         SIZE = (100, 100)
         # straigh forward case: RGB and JPEG
         COLOUR = (0, 127, 255)
-        im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR)
+        im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)
         converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
-        self.assertEqual(converted.mode, 'RGB')
+        self.assertEqual(converted.mode, "RGB")
         self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
         # check that we don't convert JPEGs again
         self.assertEqual(converted_buf, buf)
 
         # check that thumbnail keep image ratio
-        thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf)
-        self.assertEqual(thumbnail.mode, 'RGB')
+        thumbnail, _ = self.pipeline.convert_image(
+            converted, size=(10, 25), response_body=converted_buf
+        )
+        self.assertEqual(thumbnail.mode, "RGB")
         self.assertEqual(thumbnail.size, (10, 10))
 
         # transparency case: RGBA and PNG
         COLOUR = (0, 127, 255, 50)
-        im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR)
+        im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
         converted, _ = self.pipeline.convert_image(im, response_body=buf)
-        self.assertEqual(converted.mode, 'RGB')
+        self.assertEqual(converted.mode, "RGB")
         self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
         # transparency case with palette: P and PNG
         COLOUR = (0, 127, 255, 50)
-        im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR)
-        im = im.convert('P')
+        im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
+        im = im.convert("P")
         converted, _ = self.pipeline.convert_image(im, response_body=buf)
-        self.assertEqual(converted.mode, 'RGB')
+        self.assertEqual(converted.mode, "RGB")
         self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
 
@@ -252,11 +325,11 @@ class DeprecatedImagesPipeline(ImagesPipeline):
 
     def image_key(self, url):
         image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
-        return f'empty/{image_guid}.jpg'
+        return f"empty/{image_guid}.jpg"
 
     def thumb_key(self, url, thumb_id):
         thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
-        return f'thumbsup/{thumb_id}/{thumb_guid}.jpg'
+        return f"thumbsup/{thumb_id}/{thumb_guid}.jpg"
 
 
 class ImagesPipelineTestCaseFieldsMixin:
@@ -264,35 +337,43 @@ class ImagesPipelineTestCaseFieldsMixin:
     skip = skip_pillow
 
     def test_item_fields_default(self):
-        url = 'http://www.example.com/images/1.jpg'
-        item = self.item_class(name='item1', image_urls=[url])
-        pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
+        url = "http://www.example.com/images/1.jpg"
+        item = self.item_class(name="item1", image_urls=[url])
+        pipeline = ImagesPipeline.from_settings(
+            Settings({"IMAGES_STORE": "s3://example/images/"})
+        )
         requests = list(pipeline.get_media_requests(item, None))
         self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
+        results = [(True, {"url": url})]
         item = pipeline.item_completed(results, item, None)
         images = ItemAdapter(item).get("images")
         self.assertEqual(images, [results[0][1]])
         self.assertIsInstance(item, self.item_class)
 
     def test_item_fields_override_settings(self):
-        url = 'http://www.example.com/images/1.jpg'
-        item = self.item_class(name='item1', custom_image_urls=[url])
-        pipeline = ImagesPipeline.from_settings(Settings({
-            'IMAGES_STORE': 's3://example/images/',
-            'IMAGES_URLS_FIELD': 'custom_image_urls',
-            'IMAGES_RESULT_FIELD': 'custom_images'
-        }))
+        url = "http://www.example.com/images/1.jpg"
+        item = self.item_class(name="item1", custom_image_urls=[url])
+        pipeline = ImagesPipeline.from_settings(
+            Settings(
+                {
+                    "IMAGES_STORE": "s3://example/images/",
+                    "IMAGES_URLS_FIELD": "custom_image_urls",
+                    "IMAGES_RESULT_FIELD": "custom_images",
+                }
+            )
+        )
         requests = list(pipeline.get_media_requests(item, None))
         self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
+        results = [(True, {"url": url})]
         item = pipeline.item_completed(results, item, None)
         custom_images = ItemAdapter(item).get("custom_images")
         self.assertEqual(custom_images, [results[0][1]])
         self.assertIsInstance(item, self.item_class)
 
 
-class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsDict(
+    ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = dict
 
 
@@ -306,7 +387,9 @@ class ImagesPipelineTestItem(Item):
     custom_images = Field()
 
 
-class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsItem(
+    ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = ImagesPipelineTestItem
 
 
@@ -321,7 +404,9 @@ class ImagesPipelineTestDataClass:
     custom_images: list = dataclasses.field(default_factory=list)
 
 
-class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsDataClass(
+    ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = ImagesPipelineTestDataClass
 
 
@@ -336,7 +421,9 @@ class ImagesPipelineTestAttrsItem:
     custom_images = attr.ib(default=lambda: [])
 
 
-class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsAttrsItem(
+    ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
     item_class = ImagesPipelineTestAttrsItem
 
 
@@ -351,7 +438,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         ("MIN_HEIGHT", "IMAGES_MIN_HEIGHT"),
         ("IMAGES_URLS_FIELD", "IMAGES_URLS_FIELD"),
         ("IMAGES_RESULT_FIELD", "IMAGES_RESULT_FIELD"),
-        ("THUMBS", "IMAGES_THUMBS")
+        ("THUMBS", "IMAGES_THUMBS"),
     ]
 
     # This should match what is defined in ImagesPipeline.
@@ -360,8 +447,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         MIN_HEIGHT=0,
         EXPIRES=90,
         THUMBS={},
-        IMAGES_URLS_FIELD='image_urls',
-        IMAGES_RESULT_FIELD='images'
+        IMAGES_URLS_FIELD="image_urls",
+        IMAGES_RESULT_FIELD="images",
     )
 
     def setUp(self):
@@ -387,26 +474,30 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
             "IMAGES_MIN_WIDTH": random.randint(1, 1000),
             "IMAGES_MIN_HEIGHT": random.randint(1, 1000),
             "IMAGES_THUMBS": {
-                'small': (random.randint(1, 1000), random.randint(1, 1000)),
-                'big': (random.randint(1, 1000), random.randint(1, 1000))
-            }
+                "small": (random.randint(1, 1000), random.randint(1, 1000)),
+                "big": (random.randint(1, 1000), random.randint(1, 1000)),
+            },
         }
         if not prefix:
             return settings
 
-        return {prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v for k, v in settings.items()}
+        return {
+            prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v
+            for k, v in settings.items()
+        }
 
     def _generate_fake_pipeline_subclass(self):
         """
         :return: ImagePipeline class will all uppercase attributes set.
         """
+
         class UserDefinedImagePipeline(ImagesPipeline):
             # Values should be in different range than fake_settings.
             MIN_WIDTH = random.randint(1000, 2000)
             MIN_HEIGHT = random.randint(1000, 2000)
             THUMBS = {
-                'small': (random.randint(1000, 2000), random.randint(1000, 2000)),
-                'big': (random.randint(1000, 2000), random.randint(1000, 2000))
+                "small": (random.randint(1000, 2000), random.randint(1000, 2000)),
+                "big": (random.randint(1000, 2000), random.randint(1000, 2000)),
             }
             EXPIRES = random.randint(1000, 2000)
             IMAGES_URLS_FIELD = "field_one"
@@ -427,7 +518,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
             expected_default_value = self.default_pipeline_settings.get(pipe_attr)
             custom_value = custom_settings.get(settings_attr)
             self.assertNotEqual(expected_default_value, custom_value)
-            self.assertEqual(getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value)
+            self.assertEqual(
+                getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value
+            )
             self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value)
 
     def test_subclass_attrs_preserved_default_settings(self):
@@ -464,10 +557,13 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         If there are no settings for subclass and no subclass attributes, pipeline should use
         attributes of base class.
         """
+
         class UserDefinedImagePipeline(ImagesPipeline):
             pass
 
-        user_pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+        user_pipeline = UserDefinedImagePipeline.from_settings(
+            Settings({"IMAGES_STORE": self.tempdir})
+        )
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
             # Values from settings for custom pipeline should be set on pipeline instance.
             custom_value = self.default_pipeline_settings.get(pipe_attr.upper())
@@ -478,6 +574,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         If there are custom settings for subclass and NO class attributes, pipeline should use custom
         settings.
         """
+
         class UserDefinedImagePipeline(ImagesPipeline):
             pass
 
@@ -508,7 +605,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         class UserDefinedImagePipeline(ImagesPipeline):
             DEFAULT_IMAGES_URLS_FIELD = "something"
             DEFAULT_IMAGES_RESULT_FIELD = "something_else"
-        pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+
+        pipeline = UserDefinedImagePipeline.from_settings(
+            Settings({"IMAGES_STORE": self.tempdir})
+        )
         self.assertEqual(pipeline.images_result_field, "something_else")
         self.assertEqual(pipeline.images_urls_field, "something")
 
@@ -526,12 +626,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
 
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
             expected_value = settings.get(settings_attr)
-            self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()),
-                             expected_value)
+            self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value)
 
 
 class NoimagesDropTestCase(unittest.TestCase):
-
     def test_deprecation_warning(self):
         arg = str()
         with warnings.catch_warnings(record=True) as w:
@@ -539,8 +637,10 @@ class NoimagesDropTestCase(unittest.TestCase):
             self.assertEqual(len(w), 1)
             self.assertEqual(w[0].category, ScrapyDeprecationWarning)
         with warnings.catch_warnings(record=True) as w:
+
             class SubclassedNoimagesDrop(NoimagesDrop):
                 pass
+
             SubclassedNoimagesDrop(arg)
             self.assertEqual(len(w), 1)
             self.assertEqual(w[0].category, ScrapyDeprecationWarning)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 0a94ae699..87ab03395 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -23,13 +23,15 @@ from scrapy.utils.test import get_crawler
 try:
     from PIL import Image  # noqa: imported just to check for the import error
 except ImportError:
-    skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+    skip_pillow: Optional[
+        str
+    ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
 else:
     skip_pillow = None
 
 
 def _mocked_download_func(request, info):
-    response = request.meta.get('response')
+    response = request.meta.get("response")
     return response() if callable(response) else response
 
 
@@ -40,7 +42,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     def setUp(self):
         spider_cls = Spider
-        self.spider = spider_cls('media.com')
+        self.spider = spider_cls("media.com")
         crawler = get_crawler(spider_cls, self.settings)
         self.pipe = self.pipeline_class.from_crawler(crawler)
         self.pipe.download_func = _mocked_download_func
@@ -50,29 +52,29 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     def tearDown(self):
         for name, signal in vars(signals).items():
-            if not name.startswith('_'):
+            if not name.startswith("_"):
                 disconnect_all(signal)
 
     def test_default_media_to_download(self):
-        request = Request('http://url')
+        request = Request("http://url")
         assert self.pipe.media_to_download(request, self.info) is None
 
     def test_default_get_media_requests(self):
-        item = dict(name='name')
+        item = dict(name="name")
         assert self.pipe.get_media_requests(item, self.info) is None
 
     def test_default_media_downloaded(self):
-        request = Request('http://url')
-        response = Response('http://url', body=b'')
+        request = Request("http://url")
+        response = Response("http://url", body=b"")
         assert self.pipe.media_downloaded(response, request, self.info) is response
 
     def test_default_media_failed(self):
-        request = Request('http://url')
+        request = Request("http://url")
         fail = Failure(Exception())
         assert self.pipe.media_failed(fail, request, self.info) is fail
 
     def test_default_item_completed(self):
-        item = dict(name='name')
+        item = dict(name="name")
         assert self.pipe.item_completed([], item, self.info) is item
 
         # Check that failures are logged by default
@@ -85,7 +87,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         assert new_item is item
         assert len(log.records) == 1
         record = log.records[0]
-        assert record.levelname == 'ERROR'
+        assert record.levelname == "ERROR"
         self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
 
         # disable failure logging and check again
@@ -97,14 +99,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     @inlineCallbacks
     def test_default_process_item(self):
-        item = dict(name='name')
+        item = dict(name="name")
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
 
     def test_modify_media_request(self):
-        request = Request('http://url')
+        request = Request("http://url")
         self.pipe._modify_media_request(request)
-        assert request.meta == {'handle_httpstatus_all': True}
+        assert request.meta == {"handle_httpstatus_all": True}
 
     def test_should_remove_req_res_references_before_caching_the_results(self):
         """Regression test case to prevent a memory leak in the Media Pipeline.
@@ -134,8 +136,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         Chaining (https://www.python.org/dev/peps/pep-3134/).
         """
         # Create sample pair of Request and Response objects
-        request = Request('http://url')
-        response = Response('http://url', body=b'', request=request)
+        request = Request("http://url")
+        response = Response("http://url", body=b"", request=request)
 
         # Simulate the Media Pipeline behavior to produce a Twisted Failure
         try:
@@ -146,7 +148,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
             try:
                 # Simulate the media_downloaded callback raising a FileException
                 # This usually happens when the status code is not 200 OK
-                raise FileException('download-error')
+                raise FileException("download-error")
             except Exception as exc:
                 file_exc = exc
                 # Simulate Twisted capturing the FileException
@@ -171,42 +173,41 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         # ... encapsulating the original FileException ...
         self.assertEqual(info.downloaded[fp].value, file_exc)
         # ... but it should not store the StopIteration exception on its context
-        context = getattr(info.downloaded[fp].value, '__context__', None)
+        context = getattr(info.downloaded[fp].value, "__context__", None)
         self.assertIsNone(context)
 
 
 class MockedMediaPipeline(MediaPipeline):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def download(self, request, info):
-        self._mockcalled.append('download')
+        self._mockcalled.append("download")
         return super().download(request, info)
 
     def media_to_download(self, request, info, *, item=None):
-        self._mockcalled.append('media_to_download')
-        if 'result' in request.meta:
-            return request.meta.get('result')
+        self._mockcalled.append("media_to_download")
+        if "result" in request.meta:
+            return request.meta.get("result")
         return super().media_to_download(request, info)
 
     def get_media_requests(self, item, info):
-        self._mockcalled.append('get_media_requests')
-        return item.get('requests')
+        self._mockcalled.append("get_media_requests")
+        return item.get("requests")
 
     def media_downloaded(self, response, request, info, *, item=None):
-        self._mockcalled.append('media_downloaded')
+        self._mockcalled.append("media_downloaded")
         return super().media_downloaded(response, request, info)
 
     def media_failed(self, failure, request, info):
-        self._mockcalled.append('media_failed')
+        self._mockcalled.append("media_failed")
         return super().media_failed(failure, request, info)
 
     def item_completed(self, results, item, info):
-        self._mockcalled.append('item_completed')
+        self._mockcalled.append("item_completed")
         item = super().item_completed(results, item, info)
-        item['results'] = results
+        item["results"] = results
         return item
 
 
@@ -215,72 +216,94 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
     pipeline_class = MockedMediaPipeline
 
     def _callback(self, result):
-        self.pipe._mockcalled.append('request_callback')
+        self.pipe._mockcalled.append("request_callback")
         return result
 
     def _errback(self, result):
-        self.pipe._mockcalled.append('request_errback')
+        self.pipe._mockcalled.append("request_errback")
         return result
 
     @inlineCallbacks
     def test_result_succeed(self):
-        rsp = Response('http://url1')
-        req = Request('http://url1', meta=dict(response=rsp),
-                      callback=self._callback, errback=self._errback)
+        rsp = Response("http://url1")
+        req = Request(
+            "http://url1",
+            meta=dict(response=rsp),
+            callback=self._callback,
+            errback=self._errback,
+        )
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item['results'], [(True, rsp)])
+        self.assertEqual(new_item["results"], [(True, rsp)])
         self.assertEqual(
             self.pipe._mockcalled,
-            ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
+            [
+                "get_media_requests",
+                "media_to_download",
+                "media_downloaded",
+                "request_callback",
+                "item_completed",
+            ],
+        )
 
     @inlineCallbacks
     def test_result_failure(self):
         self.pipe.LOG_FAILED_RESULTS = False
         fail = Failure(Exception())
-        req = Request('http://url1', meta=dict(response=fail),
-                      callback=self._callback, errback=self._errback)
+        req = Request(
+            "http://url1",
+            meta=dict(response=fail),
+            callback=self._callback,
+            errback=self._errback,
+        )
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item['results'], [(False, fail)])
+        self.assertEqual(new_item["results"], [(False, fail)])
         self.assertEqual(
             self.pipe._mockcalled,
-            ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
+            [
+                "get_media_requests",
+                "media_to_download",
+                "media_failed",
+                "request_errback",
+                "item_completed",
+            ],
+        )
 
     @inlineCallbacks
     def test_mix_of_success_and_failure(self):
         self.pipe.LOG_FAILED_RESULTS = False
-        rsp1 = Response('http://url1')
-        req1 = Request('http://url1', meta=dict(response=rsp1))
+        rsp1 = Response("http://url1")
+        req1 = Request("http://url1", meta=dict(response=rsp1))
         fail = Failure(Exception())
-        req2 = Request('http://url2', meta=dict(response=fail))
+        req2 = Request("http://url2", meta=dict(response=fail))
         item = dict(requests=[req1, req2])
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
+        self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)])
         m = self.pipe._mockcalled
         # only once
-        self.assertEqual(m[0], 'get_media_requests')  # first hook called
-        self.assertEqual(m.count('get_media_requests'), 1)
-        self.assertEqual(m.count('item_completed'), 1)
-        self.assertEqual(m[-1], 'item_completed')  # last hook called
+        self.assertEqual(m[0], "get_media_requests")  # first hook called
+        self.assertEqual(m.count("get_media_requests"), 1)
+        self.assertEqual(m.count("item_completed"), 1)
+        self.assertEqual(m[-1], "item_completed")  # last hook called
         # twice, one per request
-        self.assertEqual(m.count('media_to_download'), 2)
+        self.assertEqual(m.count("media_to_download"), 2)
         # one to handle success and other for failure
-        self.assertEqual(m.count('media_downloaded'), 1)
-        self.assertEqual(m.count('media_failed'), 1)
+        self.assertEqual(m.count("media_downloaded"), 1)
+        self.assertEqual(m.count("media_failed"), 1)
 
     @inlineCallbacks
     def test_get_media_requests(self):
         # returns single Request (without callback)
-        req = Request('http://url')
+        req = Request("http://url")
         item = dict(requests=req)  # pass a single item
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
         self.assertIn(self.fingerprint(req), self.info.downloaded)
 
         # returns iterable of Requests
-        req1 = Request('http://url1')
-        req2 = Request('http://url2')
+        req1 = Request("http://url1")
+        req2 = Request("http://url2")
         item = dict(requests=iter([req1, req2]))
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
@@ -289,30 +312,34 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
 
     @inlineCallbacks
     def test_results_are_cached_across_multiple_items(self):
-        rsp1 = Response('http://url1')
-        req1 = Request('http://url1', meta=dict(response=rsp1))
+        rsp1 = Response("http://url1")
+        req1 = Request("http://url1", meta=dict(response=rsp1))
         item = dict(requests=req1)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
-        self.assertEqual(new_item['results'], [(True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, rsp1)])
 
         # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same
-        req2 = Request(req1.url, meta=dict(response=Response('http://donot.download.me')))
+        req2 = Request(
+            req1.url, meta=dict(response=Response("http://donot.download.me"))
+        )
         item = dict(requests=req2)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
         self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
-        self.assertEqual(new_item['results'], [(True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, rsp1)])
 
     @inlineCallbacks
     def test_results_are_cached_for_requests_of_single_item(self):
-        rsp1 = Response('http://url1')
-        req1 = Request('http://url1', meta=dict(response=rsp1))
-        req2 = Request(req1.url, meta=dict(response=Response('http://donot.download.me')))
+        rsp1 = Response("http://url1")
+        req1 = Request("http://url1", meta=dict(response=rsp1))
+        req2 = Request(
+            req1.url, meta=dict(response=Response("http://donot.download.me"))
+        )
         item = dict(requests=[req1, req2])
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
-        self.assertEqual(new_item['results'], [(True, rsp1), (True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
 
     @inlineCallbacks
     def test_wait_if_request_is_downloading(self):
@@ -324,78 +351,80 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
             self.assertEqual(len(self.info.waiting[fp]), 2)
             return response
 
-        rsp1 = Response('http://url')
+        rsp1 = Response("http://url")
 
         def rsp1_func():
             dfd = Deferred().addCallback(_check_downloading)
-            reactor.callLater(.1, dfd.callback, rsp1)
+            reactor.callLater(0.1, dfd.callback, rsp1)
             return dfd
 
         def rsp2_func():
-            self.fail('it must cache rsp1 result and must not try to redownload')
+            self.fail("it must cache rsp1 result and must not try to redownload")
 
-        req1 = Request('http://url', meta=dict(response=rsp1_func))
+        req1 = Request("http://url", meta=dict(response=rsp1_func))
         req2 = Request(req1.url, meta=dict(response=rsp2_func))
         item = dict(requests=[req1, req2])
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item['results'], [(True, rsp1), (True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
 
     @inlineCallbacks
     def test_use_media_to_download_result(self):
-        req = Request('http://url', meta=dict(result='ITSME', response=self.fail))
+        req = Request("http://url", meta=dict(result="ITSME", response=self.fail))
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item['results'], [(True, 'ITSME')])
+        self.assertEqual(new_item["results"], [(True, "ITSME")])
         self.assertEqual(
             self.pipe._mockcalled,
-            ['get_media_requests', 'media_to_download', 'item_completed'])
+            ["get_media_requests", "media_to_download", "item_completed"],
+        )
 
 
 class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def get_media_requests(self, item, info):
-        item_url = item['image_urls'][0]
+        item_url = item["image_urls"][0]
         output_img = io.BytesIO()
-        img = Image.new('RGB', (60, 30), color='red')
-        img.save(output_img, format='JPEG')
+        img = Image.new("RGB", (60, 30), color="red")
+        img.save(output_img, format="JPEG")
         return Request(
             item_url,
-            meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
+            meta={
+                "response": Response(item_url, status=200, body=output_img.getvalue())
+            },
         )
 
     def inc_stats(self, *args, **kwargs):
         return True
 
     def media_to_download(self, request, info):
-        self._mockcalled.append('media_to_download')
+        self._mockcalled.append("media_to_download")
         return super().media_to_download(request, info)
 
     def media_downloaded(self, response, request, info):
-        self._mockcalled.append('media_downloaded')
+        self._mockcalled.append("media_downloaded")
         return super().media_downloaded(response, request, info)
 
     def file_downloaded(self, response, request, info):
-        self._mockcalled.append('file_downloaded')
+        self._mockcalled.append("file_downloaded")
         return super().file_downloaded(response, request, info)
 
     def file_path(self, request, response=None, info=None):
-        self._mockcalled.append('file_path')
+        self._mockcalled.append("file_path")
         return super().file_path(request, response, info)
 
     def thumb_path(self, request, thumb_id, response=None, info=None):
-        self._mockcalled.append('thumb_path')
+        self._mockcalled.append("thumb_path")
         return super().thumb_path(request, thumb_id, response, info)
 
     def get_images(self, response, request, info):
-        self._mockcalled.append('get_images')
+        self._mockcalled.append("get_images")
         return super().get_images(response, request, info)
 
     def image_downloaded(self, response, request, info):
-        self._mockcalled.append('image_downloaded')
+        self._mockcalled.append("image_downloaded")
         return super().image_downloaded(response, request, info)
 
 
@@ -404,20 +433,23 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
 
     def setUp(self):
         settings_dict = {
-            'IMAGES_STORE': 'store-uri',
-            'IMAGES_THUMBS': {'small': (50, 50)},
+            "IMAGES_STORE": "store-uri",
+            "IMAGES_THUMBS": {"small": (50, 50)},
         }
         crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
         self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
         self.pipe.download_func = _mocked_download_func
         self.pipe.open_spider(None)
-        self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
+        self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[])
 
     def _assert_method_called_with_warnings(self, method, message, warnings):
         self.assertIn(method, self.pipe._mockcalled)
         warningShown = False
         for warning in warnings:
-            if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning:
+            if (
+                warning["message"] == message
+                and warning["category"] == ScrapyDeprecationWarning
+            ):
                 warningShown = True
         self.assertTrue(warningShown)
 
@@ -426,125 +458,111 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'media_to_download(self, request, info) is deprecated, '
-            'please use media_to_download(self, request, info, *, item=None)'
+            "media_to_download(self, request, info) is deprecated, "
+            "please use media_to_download(self, request, info, *, item=None)"
         )
-        self._assert_method_called_with_warnings('media_to_download', message, warnings)
+        self._assert_method_called_with_warnings("media_to_download", message, warnings)
 
     @inlineCallbacks
     def test_media_downloaded_called(self):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'media_downloaded(self, response, request, info) is deprecated, '
-            'please use media_downloaded(self, response, request, info, *, item=None)'
+            "media_downloaded(self, response, request, info) is deprecated, "
+            "please use media_downloaded(self, response, request, info, *, item=None)"
         )
-        self._assert_method_called_with_warnings('media_downloaded', message, warnings)
+        self._assert_method_called_with_warnings("media_downloaded", message, warnings)
 
     @inlineCallbacks
     def test_file_downloaded_called(self):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'file_downloaded(self, response, request, info) is deprecated, '
-            'please use file_downloaded(self, response, request, info, *, item=None)'
+            "file_downloaded(self, response, request, info) is deprecated, "
+            "please use file_downloaded(self, response, request, info, *, item=None)"
         )
-        self._assert_method_called_with_warnings('file_downloaded', message, warnings)
+        self._assert_method_called_with_warnings("file_downloaded", message, warnings)
 
     @inlineCallbacks
     def test_file_path_called(self):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'file_path(self, request, response=None, info=None) is deprecated, '
-            'please use file_path(self, request, response=None, info=None, *, item=None)'
+            "file_path(self, request, response=None, info=None) is deprecated, "
+            "please use file_path(self, request, response=None, info=None, *, item=None)"
         )
-        self._assert_method_called_with_warnings('file_path', message, warnings)
+        self._assert_method_called_with_warnings("file_path", message, warnings)
 
     @inlineCallbacks
     def test_thumb_path_called(self):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
-            'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
+            "thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, "
+            "please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)"
         )
-        self._assert_method_called_with_warnings('thumb_path', message, warnings)
+        self._assert_method_called_with_warnings("thumb_path", message, warnings)
 
     @inlineCallbacks
     def test_get_images_called(self):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'get_images(self, response, request, info) is deprecated, '
-            'please use get_images(self, response, request, info, *, item=None)'
+            "get_images(self, response, request, info) is deprecated, "
+            "please use get_images(self, response, request, info, *, item=None)"
         )
-        self._assert_method_called_with_warnings('get_images', message, warnings)
+        self._assert_method_called_with_warnings("get_images", message, warnings)
 
     @inlineCallbacks
     def test_image_downloaded_called(self):
         yield self.pipe.process_item(self.item, None)
         warnings = self.flushWarnings([MediaPipeline._compatible])
         message = (
-            'image_downloaded(self, response, request, info) is deprecated, '
-            'please use image_downloaded(self, response, request, info, *, item=None)'
+            "image_downloaded(self, response, request, info) is deprecated, "
+            "please use image_downloaded(self, response, request, info, *, item=None)"
         )
-        self._assert_method_called_with_warnings('image_downloaded', message, warnings)
+        self._assert_method_called_with_warnings("image_downloaded", message, warnings)
 
 
 class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
-
     def _assert_request_no3xx(self, pipeline_class, settings):
         pipe = pipeline_class(settings=Settings(settings))
-        request = Request('http://url')
+        request = Request("http://url")
         pipe._modify_media_request(request)
 
-        self.assertIn('handle_httpstatus_list', request.meta)
+        self.assertIn("handle_httpstatus_list", request.meta)
         for status, check in [
-                (200, True),
-
-                # These are the status codes we want
-                # the downloader to handle itself
-                (301, False),
-                (302, False),
-                (302, False),
-                (307, False),
-                (308, False),
-
-                # we still want to get 4xx and 5xx
-                (400, True),
-                (404, True),
-                (500, True)]:
+            (200, True),
+            # These are the status codes we want
+            # the downloader to handle itself
+            (301, False),
+            (302, False),
+            (302, False),
+            (307, False),
+            (308, False),
+            # we still want to get 4xx and 5xx
+            (400, True),
+            (404, True),
+            (500, True),
+        ]:
             if check:
-                self.assertIn(status, request.meta['handle_httpstatus_list'])
+                self.assertIn(status, request.meta["handle_httpstatus_list"])
             else:
-                self.assertNotIn(status, request.meta['handle_httpstatus_list'])
+                self.assertNotIn(status, request.meta["handle_httpstatus_list"])
 
     def test_standard_setting(self):
-        self._assert_request_no3xx(
-            MediaPipeline,
-            {
-                'MEDIA_ALLOW_REDIRECTS': True
-            })
+        self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
 
     def test_subclass_standard_setting(self):
-
         class UserDefinedPipeline(MediaPipeline):
             pass
 
-        self._assert_request_no3xx(
-            UserDefinedPipeline,
-            {
-                'MEDIA_ALLOW_REDIRECTS': True
-            })
+        self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
 
     def test_subclass_specific_setting(self):
-
         class UserDefinedPipeline(MediaPipeline):
             pass
 
         self._assert_request_no3xx(
-            UserDefinedPipeline,
-            {
-                'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True
-            })
+            UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True}
+        )
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 8e432b913..7b905d321 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -14,13 +14,13 @@ from tests.mockserver import MockServer
 
 class SimplePipeline:
     def process_item(self, item, spider):
-        item['pipeline_passed'] = True
+        item["pipeline_passed"] = True
         return item
 
 
 class DeferredPipeline:
     def cb(self, item):
-        item['pipeline_passed'] = True
+        item["pipeline_passed"] = True
         return item
 
     def process_item(self, item, spider):
@@ -34,9 +34,10 @@ class AsyncDefPipeline:
     async def process_item(self, item, spider):
         d = Deferred()
         from twisted.internet import reactor
+
         reactor.callLater(0, d.callback, None)
         await maybe_deferred_to_future(d)
-        item['pipeline_passed'] = True
+        item["pipeline_passed"] = True
         return item
 
 
@@ -44,10 +45,11 @@ class AsyncDefAsyncioPipeline:
     async def process_item(self, item, spider):
         d = Deferred()
         from twisted.internet import reactor
+
         reactor.callLater(0, d.callback, None)
         await deferred_to_future(d)
         await asyncio.sleep(0.2)
-        item['pipeline_passed'] = await get_from_asyncio_queue(True)
+        item["pipeline_passed"] = await get_from_asyncio_queue(True)
         return item
 
 
@@ -55,23 +57,24 @@ class AsyncDefNotAsyncioPipeline:
     async def process_item(self, item, spider):
         d1 = Deferred()
         from twisted.internet import reactor
+
         reactor.callLater(0, d1.callback, None)
         await d1
         d2 = Deferred()
         reactor.callLater(0, d2.callback, None)
         await maybe_deferred_to_future(d2)
-        item['pipeline_passed'] = True
+        item["pipeline_passed"] = True
         return item
 
 
 class ItemSpider(Spider):
-    name = 'itemspider'
+    name = "itemspider"
 
     def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
+        yield Request(self.mockserver.url("/status?n=200"))
 
     def parse(self, response):
-        return {'field': 42}
+        return {"field": 42}
 
 
 class PipelineTestCase(unittest.TestCase):
@@ -84,12 +87,12 @@ class PipelineTestCase(unittest.TestCase):
 
     def _on_item_scraped(self, item):
         self.assertIsInstance(item, dict)
-        self.assertTrue(item.get('pipeline_passed'))
+        self.assertTrue(item.get("pipeline_passed"))
         self.items.append(item)
 
     def _create_crawler(self, pipeline_class):
         settings = {
-            'ITEM_PIPELINES': {pipeline_class: 1},
+            "ITEM_PIPELINES": {pipeline_class: 1},
         }
         crawler = get_crawler(ItemSpider, settings)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index ec55033d1..96a64c19d 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -19,7 +19,9 @@ class PriorityQueueTest(unittest.TestCase):
 
     def test_queue_push_pop_one(self):
         temp_dir = tempfile.mkdtemp()
-        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+        queue = ScrapyPriorityQueue.from_crawler(
+            self.crawler, FifoMemoryQueue, temp_dir
+        )
         self.assertIsNone(queue.pop())
         self.assertEqual(len(queue), 0)
         req1 = Request("https://example.org/1", priority=1)
@@ -35,9 +37,14 @@ class PriorityQueueTest(unittest.TestCase):
         if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
             raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
         temp_dir = tempfile.mkdtemp()
-        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+        queue = ScrapyPriorityQueue.from_crawler(
+            self.crawler, FifoMemoryQueue, temp_dir
+        )
         queue.push(Request("https://example.org"))
-        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+        with self.assertRaises(
+            NotImplementedError,
+            msg="The underlying queue class does not implement 'peek'",
+        ):
             queue.peek()
         queue.close()
 
@@ -45,7 +52,9 @@ class PriorityQueueTest(unittest.TestCase):
         if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
             raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined")
         temp_dir = tempfile.mkdtemp()
-        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+        queue = ScrapyPriorityQueue.from_crawler(
+            self.crawler, FifoMemoryQueue, temp_dir
+        )
         self.assertEqual(len(queue), 0)
         self.assertIsNone(queue.peek())
         req1 = Request("https://example.org/1")
@@ -67,7 +76,9 @@ class PriorityQueueTest(unittest.TestCase):
 
     def test_queue_push_pop_priorities(self):
         temp_dir = tempfile.mkdtemp()
-        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3])
+        queue = ScrapyPriorityQueue.from_crawler(
+            self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
+        )
         self.assertIsNone(queue.pop())
         self.assertEqual(len(queue), 0)
         req1 = Request("https://example.org/1", priority=1)
@@ -119,7 +130,10 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase):
         if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
             raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
         self.queue.push(Request("https://example.org"))
-        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+        with self.assertRaises(
+            NotImplementedError,
+            msg="The underlying queue class does not implement 'peek'",
+        ):
             self.queue.peek()
 
     def test_peek(self):
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ea7701b5d..5aeae7546 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -17,30 +17,40 @@ from tests.spiders import SimpleSpider, SingleRequestSpider
 
 
 class MitmProxy:
-    auth_user = 'scrapy'
-    auth_pass = 'scrapy'
+    auth_user = "scrapy"
+    auth_pass = "scrapy"
 
     def start(self):
         from scrapy.utils.test import get_testenv
+
         script = """
 import sys
 from mitmproxy.tools.main import mitmdump
 sys.argv[0] = "mitmdump"
 sys.exit(mitmdump())
         """
-        cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem'
-        self.proc = Popen([sys.executable,
-                           '-c', script,
-                           '--listen-host', '127.0.0.1',
-                           '--listen-port', '0',
-                           '--proxyauth', f'{self.auth_user}:{self.auth_pass}',
-                           '--certs', str(cert_path),
-                           '--ssl-insecure',
-                           ],
-                          stdout=PIPE, env=get_testenv())
-        line = self.proc.stdout.readline().decode('utf-8')
-        host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
-        address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}'
+        cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem"
+        self.proc = Popen(
+            [
+                sys.executable,
+                "-c",
+                script,
+                "--listen-host",
+                "127.0.0.1",
+                "--listen-port",
+                "0",
+                "--proxyauth",
+                f"{self.auth_user}:{self.auth_pass}",
+                "--certs",
+                str(cert_path),
+                "--ssl-insecure",
+            ],
+            stdout=PIPE,
+            env=get_testenv(),
+        )
+        line = self.proc.stdout.readline().decode("utf-8")
+        host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+        address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
         return address
 
     def stop(self):
@@ -50,17 +60,16 @@ sys.exit(mitmdump())
 
 def _wrong_credentials(proxy_url):
     bad_auth_proxy = list(urlsplit(proxy_url))
-    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")
     return urlunsplit(bad_auth_proxy)
 
 
 class ProxyConnectTestCase(TestCase):
-
     def setUp(self):
         try:
             import mitmproxy  # noqa: F401
         except ImportError:
-            self.skipTest('mitmproxy is not installed')
+            self.skipTest("mitmproxy is not installed")
 
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -68,8 +77,8 @@ class ProxyConnectTestCase(TestCase):
 
         self._proxy = MitmProxy()
         proxy_url = self._proxy.start()
-        os.environ['https_proxy'] = proxy_url
-        os.environ['http_proxy'] = proxy_url
+        os.environ["https_proxy"] = proxy_url
+        os.environ["http_proxy"] = proxy_url
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
@@ -85,7 +94,7 @@ class ProxyConnectTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+        os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
@@ -100,13 +109,13 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as log:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, log)
-        echo = json.loads(crawler.spider.meta['responses'][0].text)
-        self.assertTrue('Proxy-Authorization' not in echo['headers'])
+        echo = json.loads(crawler.spider.meta["responses"][0].text)
+        self.assertTrue("Proxy-Authorization" not in echo["headers"])
 
     def _assert_got_response_code(self, code, log):
         print(log)
-        self.assertEqual(str(log).count(f'Crawled ({code})'), 1)
+        self.assertEqual(str(log).count(f"Crawled ({code})"), 1)
 
     def _assert_got_tunnel_error(self, log):
         print(log)
-        self.assertIn('TunnelError', str(log))
+        self.assertIn("TunnelError", str(log))
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 15e400327..f7540743d 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -60,7 +60,6 @@ class AlternativeCallbacksMiddleware:
 
 
 class CrawlTestCase(TestCase):
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -90,11 +89,14 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_downloader_middleware_raise_exception(self):
         url = self.mockserver.url("/status?n=200")
-        crawler = get_crawler(SingleRequestSpider, {
-            "DOWNLOADER_MIDDLEWARES": {
-                RaiseExceptionRequestMiddleware: 590,
+        crawler = get_crawler(
+            SingleRequestSpider,
+            {
+                "DOWNLOADER_MIDDLEWARES": {
+                    RaiseExceptionRequestMiddleware: 590,
+                },
             },
-        })
+        )
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         failure = crawler.spider.meta["failure"]
         self.assertEqual(failure.request.url, url)
@@ -116,11 +118,14 @@ class CrawlTestCase(TestCase):
             signal_params["request"] = request
 
         url = self.mockserver.url("/status?n=200")
-        crawler = get_crawler(SingleRequestSpider, {
-            "DOWNLOADER_MIDDLEWARES": {
-                ProcessResponseMiddleware: 595,
-            }
-        })
+        crawler = get_crawler(
+            SingleRequestSpider,
+            {
+                "DOWNLOADER_MIDDLEWARES": {
+                    ProcessResponseMiddleware: 595,
+                }
+            },
+        )
         crawler.signals.connect(signal_handler, signal=signals.response_received)
 
         with LogCapture() as log:
@@ -133,7 +138,11 @@ class CrawlTestCase(TestCase):
         self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
 
         log.check_present(
-            ("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)"),
+            (
+                "scrapy.core.engine",
+                "DEBUG",
+                f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)",
+            ),
         )
 
     @defer.inlineCallbacks
@@ -145,12 +154,15 @@ class CrawlTestCase(TestCase):
         The spider callback should receive the overridden response.request
         """
         url = self.mockserver.url("/status?n=200")
-        crawler = get_crawler(SingleRequestSpider, {
-            "DOWNLOADER_MIDDLEWARES": {
-                RaiseExceptionRequestMiddleware: 590,
-                CatchExceptionOverrideRequestMiddleware: 595,
+        crawler = get_crawler(
+            SingleRequestSpider,
+            {
+                "DOWNLOADER_MIDDLEWARES": {
+                    RaiseExceptionRequestMiddleware: 590,
+                    CatchExceptionOverrideRequestMiddleware: 595,
+                },
             },
-        })
+        )
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -165,12 +177,15 @@ class CrawlTestCase(TestCase):
         The spider callback should receive the original response.request
         """
         url = self.mockserver.url("/status?n=200")
-        crawler = get_crawler(SingleRequestSpider, {
-            "DOWNLOADER_MIDDLEWARES": {
-                RaiseExceptionRequestMiddleware: 590,
-                CatchExceptionDoNotOverrideRequestMiddleware: 595,
+        crawler = get_crawler(
+            SingleRequestSpider,
+            {
+                "DOWNLOADER_MIDDLEWARES": {
+                    RaiseExceptionRequestMiddleware: 590,
+                    CatchExceptionDoNotOverrideRequestMiddleware: 595,
+                },
             },
-        })
+        )
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -182,16 +197,23 @@ class CrawlTestCase(TestCase):
         Downloader middleware which returns a response with a
         specific 'request' attribute, with an alternative callback
         """
-        crawler = get_crawler(AlternativeCallbacksSpider, {
-            "DOWNLOADER_MIDDLEWARES": {
-                AlternativeCallbacksMiddleware: 595,
-            }
-        })
+        crawler = get_crawler(
+            AlternativeCallbacksSpider,
+            {
+                "DOWNLOADER_MIDDLEWARES": {
+                    AlternativeCallbacksMiddleware: 595,
+                }
+            },
+        )
 
         with LogCapture() as log:
             url = self.mockserver.url("/status?n=200")
             yield crawler.crawl(seed=url, mockserver=self.mockserver)
 
         log.check_present(
-            ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"),
+            (
+                "alternative_callbacks_spider",
+                "INFO",
+                "alt_callback was invoked with foo=bar",
+            ),
         )
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 63e3aac00..5a3dae103 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -12,14 +12,15 @@ class InjectArgumentsDownloaderMiddleware:
     """
     Make sure downloader middlewares are able to update the keyword arguments
     """
+
     def process_request(self, request, spider):
-        if request.callback.__name__ == 'parse_downloader_mw':
-            request.cb_kwargs['from_process_request'] = True
+        if request.callback.__name__ == "parse_downloader_mw":
+            request.cb_kwargs["from_process_request"] = True
         return None
 
     def process_response(self, request, response, spider):
-        if request.callback.__name__ == 'parse_downloader_mw':
-            request.cb_kwargs['from_process_response'] = True
+        if request.callback.__name__ == "parse_downloader_mw":
+            request.cb_kwargs["from_process_response"] = True
         return response
 
 
@@ -27,32 +28,36 @@ class InjectArgumentsSpiderMiddleware:
     """
     Make sure spider middlewares are able to update the keyword arguments
     """
+
     def process_start_requests(self, start_requests, spider):
         for request in start_requests:
-            if request.callback.__name__ == 'parse_spider_mw':
-                request.cb_kwargs['from_process_start_requests'] = True
+            if request.callback.__name__ == "parse_spider_mw":
+                request.cb_kwargs["from_process_start_requests"] = True
             yield request
 
     def process_spider_input(self, response, spider):
         request = response.request
-        if request.callback.__name__ == 'parse_spider_mw':
-            request.cb_kwargs['from_process_spider_input'] = True
+        if request.callback.__name__ == "parse_spider_mw":
+            request.cb_kwargs["from_process_spider_input"] = True
         return None
 
     def process_spider_output(self, response, result, spider):
         for element in result:
-            if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':
-                element.cb_kwargs['from_process_spider_output'] = True
+            if (
+                isinstance(element, Request)
+                and element.callback.__name__ == "parse_spider_mw_2"
+            ):
+                element.cb_kwargs["from_process_spider_output"] = True
             yield element
 
 
 class KeywordArgumentsSpider(MockServerSpider):
-    name = 'kwargs'
+    name = "kwargs"
     custom_settings = {
-        'DOWNLOADER_MIDDLEWARES': {
+        "DOWNLOADER_MIDDLEWARES": {
             InjectArgumentsDownloaderMiddleware: 750,
         },
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             InjectArgumentsSpiderMiddleware: 750,
         },
     }
@@ -60,50 +65,61 @@ class KeywordArgumentsSpider(MockServerSpider):
     checks = []
 
     def start_requests(self):
-        data = {'key': 'value', 'number': 123, 'callback': 'some_callback'}
-        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
-        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
-        yield Request(self.mockserver.url('/general_without'), self.parse_general)
-        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
-        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
-        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
-        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
-        yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
-        yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)
+        data = {"key": "value", "number": 123, "callback": "some_callback"}
+        yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data)
+        yield Request(
+            self.mockserver.url("/general_with"), self.parse_general, cb_kwargs=data
+        )
+        yield Request(self.mockserver.url("/general_without"), self.parse_general)
+        yield Request(self.mockserver.url("/no_kwargs"), self.parse_no_kwargs)
+        yield Request(
+            self.mockserver.url("/default"), self.parse_default, cb_kwargs=data
+        )
+        yield Request(
+            self.mockserver.url("/takes_less"), self.parse_takes_less, cb_kwargs=data
+        )
+        yield Request(
+            self.mockserver.url("/takes_more"), self.parse_takes_more, cb_kwargs=data
+        )
+        yield Request(self.mockserver.url("/downloader_mw"), self.parse_downloader_mw)
+        yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw)
 
     def parse_first(self, response, key, number):
-        self.checks.append(key == 'value')
+        self.checks.append(key == "value")
         self.checks.append(number == 123)
-        self.crawler.stats.inc_value('boolean_checks', 2)
+        self.crawler.stats.inc_value("boolean_checks", 2)
         yield response.follow(
-            self.mockserver.url('/two'),
+            self.mockserver.url("/two"),
             self.parse_second,
-            cb_kwargs={'new_key': 'new_value'})
+            cb_kwargs={"new_key": "new_value"},
+        )
 
     def parse_second(self, response, new_key):
-        self.checks.append(new_key == 'new_value')
-        self.crawler.stats.inc_value('boolean_checks')
+        self.checks.append(new_key == "new_value")
+        self.crawler.stats.inc_value("boolean_checks")
 
     def parse_general(self, response, **kwargs):
-        if response.url.endswith('/general_with'):
-            self.checks.append(kwargs['key'] == 'value')
-            self.checks.append(kwargs['number'] == 123)
-            self.checks.append(kwargs['callback'] == 'some_callback')
-            self.crawler.stats.inc_value('boolean_checks', 3)
-        elif response.url.endswith('/general_without'):
-            self.checks.append(kwargs == {})  # pylint: disable=use-implicit-booleaness-not-comparison
-            self.crawler.stats.inc_value('boolean_checks')
+        if response.url.endswith("/general_with"):
+            self.checks.append(kwargs["key"] == "value")
+            self.checks.append(kwargs["number"] == 123)
+            self.checks.append(kwargs["callback"] == "some_callback")
+            self.crawler.stats.inc_value("boolean_checks", 3)
+        elif response.url.endswith("/general_without"):
+            self.checks.append(
+                kwargs == {}
+            )  # pylint: disable=use-implicit-booleaness-not-comparison
+            self.crawler.stats.inc_value("boolean_checks")
 
     def parse_no_kwargs(self, response):
-        self.checks.append(response.url.endswith('/no_kwargs'))
-        self.crawler.stats.inc_value('boolean_checks')
+        self.checks.append(response.url.endswith("/no_kwargs"))
+        self.crawler.stats.inc_value("boolean_checks")
 
     def parse_default(self, response, key, number=None, default=99):
-        self.checks.append(response.url.endswith('/default'))
-        self.checks.append(key == 'value')
+        self.checks.append(response.url.endswith("/default"))
+        self.checks.append(key == "value")
         self.checks.append(number == 123)
         self.checks.append(default == 99)
-        self.crawler.stats.inc_value('boolean_checks', 4)
+        self.crawler.stats.inc_value("boolean_checks", 4)
 
     def parse_takes_less(self, response, key, callback):
         """
@@ -117,20 +133,24 @@ class KeywordArgumentsSpider(MockServerSpider):
         TypeError: parse_takes_more() missing 1 required positional argument: 'other'
         """
 
-    def parse_downloader_mw(self, response, from_process_request, from_process_response):
+    def parse_downloader_mw(
+        self, response, from_process_request, from_process_response
+    ):
         self.checks.append(bool(from_process_request))
         self.checks.append(bool(from_process_response))
-        self.crawler.stats.inc_value('boolean_checks', 2)
+        self.crawler.stats.inc_value("boolean_checks", 2)
 
-    def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):
+    def parse_spider_mw(
+        self, response, from_process_spider_input, from_process_start_requests
+    ):
         self.checks.append(bool(from_process_spider_input))
         self.checks.append(bool(from_process_start_requests))
-        self.crawler.stats.inc_value('boolean_checks', 2)
-        return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)
+        self.crawler.stats.inc_value("boolean_checks", 2)
+        return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)
 
     def parse_spider_mw_2(self, response, from_process_spider_output):
         self.checks.append(bool(from_process_spider_output))
-        self.crawler.stats.inc_value('boolean_checks', 1)
+        self.crawler.stats.inc_value("boolean_checks", 1)
 
 
 class CallbackKeywordArgumentsTestCase(TestCase):
@@ -150,24 +170,26 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         with LogCapture() as log:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(all(crawler.spider.checks))
-        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+        self.assertEqual(
+            len(crawler.spider.checks), crawler.stats.get_value("boolean_checks")
+        )
         # check exceptions for argument mismatch
         exceptions = {}
         for line in log.records:
-            for key in ('takes_less', 'takes_more'):
+            for key in ("takes_less", "takes_more"):
                 if key in line.getMessage():
                     exceptions[key] = line
-        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+        self.assertEqual(exceptions["takes_less"].exc_info[0], TypeError)
         self.assertTrue(
-            str(exceptions['takes_less'].exc_info[1]).endswith(
+            str(exceptions["takes_less"].exc_info[1]).endswith(
                 "parse_takes_less() got an unexpected keyword argument 'number'"
             ),
-            msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]),
+            msg="Exception message: " + str(exceptions["takes_less"].exc_info[1]),
         )
-        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+        self.assertEqual(exceptions["takes_more"].exc_info[0], TypeError)
         self.assertTrue(
-            str(exceptions['takes_more'].exc_info[1]).endswith(
+            str(exceptions["takes_more"].exc_info[1]).endswith(
                 "parse_takes_more() missing 1 required positional argument: 'other'"
             ),
-            msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]),
+            msg="Exception message: " + str(exceptions["takes_more"].exc_info[1]),
         )
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index 5bdcb975b..d9067610e 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -14,7 +14,6 @@ class CustomRequest(Request):
 
 
 class RequestSerializationTest(unittest.TestCase):
-
     def setUp(self):
         self.spider = TestSpider()
 
@@ -29,13 +28,13 @@ class RequestSerializationTest(unittest.TestCase):
             errback=self.spider.handle_error,
             method="POST",
             body=b"some body",
-            headers={'content-encoding': 'text/html; charset=latin-1'},
-            cookies={'currency': 'руб'},
-            encoding='latin-1',
+            headers={"content-encoding": "text/html; charset=latin-1"},
+            cookies={"currency": "руб"},
+            encoding="latin-1",
             priority=20,
-            meta={'a': 'b'},
-            cb_kwargs={'k': 'v'},
-            flags=['testFlag'],
+            meta={"a": "b"},
+            cb_kwargs={"k": "v"},
+            flags=["testFlag"],
         )
         self._assert_serializes_ok(r, spider=self.spider)
 
@@ -80,46 +79,57 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r3, spider=self.spider)
 
     def test_callback_serialization(self):
-        r = Request("http://www.example.com", callback=self.spider.parse_item,
-                    errback=self.spider.handle_error)
+        r = Request(
+            "http://www.example.com",
+            callback=self.spider.parse_item,
+            errback=self.spider.handle_error,
+        )
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_reference_callback_serialization(self):
-        r = Request("http://www.example.com",
-                    callback=self.spider.parse_item_reference,
-                    errback=self.spider.handle_error_reference)
+        r = Request(
+            "http://www.example.com",
+            callback=self.spider.parse_item_reference,
+            errback=self.spider.handle_error_reference,
+        )
         self._assert_serializes_ok(r, spider=self.spider)
         request_dict = r.to_dict(spider=self.spider)
-        self.assertEqual(request_dict['callback'], 'parse_item_reference')
-        self.assertEqual(request_dict['errback'], 'handle_error_reference')
+        self.assertEqual(request_dict["callback"], "parse_item_reference")
+        self.assertEqual(request_dict["errback"], "handle_error_reference")
 
     def test_private_reference_callback_serialization(self):
-        r = Request("http://www.example.com",
-                    callback=self.spider._TestSpider__parse_item_reference,
-                    errback=self.spider._TestSpider__handle_error_reference)
+        r = Request(
+            "http://www.example.com",
+            callback=self.spider._TestSpider__parse_item_reference,
+            errback=self.spider._TestSpider__handle_error_reference,
+        )
         self._assert_serializes_ok(r, spider=self.spider)
         request_dict = r.to_dict(spider=self.spider)
-        self.assertEqual(request_dict['callback'],
-                         '_TestSpider__parse_item_reference')
-        self.assertEqual(request_dict['errback'],
-                         '_TestSpider__handle_error_reference')
+        self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference")
+        self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference")
 
     def test_private_callback_serialization(self):
-        r = Request("http://www.example.com",
-                    callback=self.spider._TestSpider__parse_item_private,
-                    errback=self.spider.handle_error)
+        r = Request(
+            "http://www.example.com",
+            callback=self.spider._TestSpider__parse_item_private,
+            errback=self.spider.handle_error,
+        )
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_mixin_private_callback_serialization(self):
-        r = Request("http://www.example.com",
-                    callback=self.spider._TestSpiderMixin__mixin_callback,
-                    errback=self.spider.handle_error)
+        r = Request(
+            "http://www.example.com",
+            callback=self.spider._TestSpiderMixin__mixin_callback,
+            errback=self.spider.handle_error,
+        )
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_delegated_callback_serialization(self):
-        r = Request("http://www.example.com",
-                    callback=self.spider.delegated_callback,
-                    errback=self.spider.handle_error)
+        r = Request(
+            "http://www.example.com",
+            callback=self.spider.delegated_callback,
+            errback=self.spider.handle_error,
+        )
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_unserializable_callback1(self):
@@ -134,14 +144,14 @@ class RequestSerializationTest(unittest.TestCase):
         """Parser method is removed or replaced dynamically."""
 
         class MySpider(Spider):
-            name = 'my_spider'
+            name = "my_spider"
 
             def parse(self, response):
                 pass
 
         spider = MySpider()
         r = Request("http://www.example.com", callback=spider.parse)
-        setattr(spider, 'parse', None)
+        setattr(spider, "parse", None)
         self.assertRaises(ValueError, r.to_dict, spider=spider)
 
     def test_callback_not_available(self):
@@ -157,9 +167,14 @@ class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):
         with warnings.catch_warnings(record=True) as caught:
             warnings.simplefilter("always")
             with suppress(KeyError):
-                del sys.modules["scrapy.utils.reqser"]  # delete module to reset the deprecation warning
+                del sys.modules[
+                    "scrapy.utils.reqser"
+                ]  # delete module to reset the deprecation warning
 
-            from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict
+            from scrapy.utils.reqser import (
+                request_from_dict as _from_dict,
+                request_to_dict as _to_dict,
+            )
 
             request_copy = _from_dict(_to_dict(request, spider), spider)
             self._assert_same_request(request, request_copy)
@@ -200,7 +215,7 @@ def private_handle_error(failure):
 
 
 class TestSpider(Spider, TestSpiderMixin):
-    name = 'test'
+    name = "test"
     parse_item_reference = parse_item
     handle_error_reference = handle_error
     __parse_item_reference = private_parse_item
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 4d4483881..d08ed0f68 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -7,12 +7,11 @@ from tests.mockserver import MockServer
 
 
 class SignalCatcherSpider(Spider):
-    name = 'signal_catcher'
+    name = "signal_catcher"
 
     def __init__(self, crawler, url, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        crawler.signals.connect(self.on_request_left,
-                                signal=request_left_downloader)
+        crawler.signals.connect(self.on_request_left, signal=request_left_downloader)
         self.caught_times = 0
         self.start_urls = [url]
 
@@ -26,7 +25,6 @@ class SignalCatcherSpider(Spider):
 
 
 class TestCatching(TestCase):
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -42,8 +40,7 @@ class TestCatching(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout(self):
-        crawler = get_crawler(SignalCatcherSpider,
-                              {'DOWNLOAD_TIMEOUT': 0.1})
+        crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
         yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
         self.assertEqual(crawler.spider.caught_times, 1)
 
@@ -56,5 +53,5 @@ class TestCatching(TestCase):
     @defer.inlineCallbacks
     def test_noconnect(self):
         crawler = get_crawler(SignalCatcherSpider)
-        yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
+        yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
         self.assertEqual(crawler.spider.caught_times, 1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 4b4095fb0..57484a2a1 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -5,15 +5,14 @@ from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Heade
 
 
 class ResponseTypesTest(unittest.TestCase):
-
     def test_from_filename(self):
         mappings = [
-            ('data.bin', Response),
-            ('file.txt', TextResponse),
-            ('file.xml.gz', Response),
-            ('file.xml', XmlResponse),
-            ('file.html', HtmlResponse),
-            ('file.unknownext', Response),
+            ("data.bin", Response),
+            ("file.txt", TextResponse),
+            ("file.xml.gz", Response),
+            ("file.xml", XmlResponse),
+            ("file.html", HtmlResponse),
+            ("file.unknownext", Response),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_filename(source)
@@ -22,13 +21,12 @@ class ResponseTypesTest(unittest.TestCase):
     def test_from_content_disposition(self):
         mappings = [
             (b'attachment; filename="data.xml"', XmlResponse),
-            (b'attachment; filename=data.xml', XmlResponse),
-            ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
-            ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
-            ('attachment;filename=data高.doc'.encode('gbk'), Response),
-            ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
-            ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
-
+            (b"attachment; filename=data.xml", XmlResponse),
+            ("attachment;filename=data£.tar.gz".encode("utf-8"), Response),
+            ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
+            ("attachment;filename=data高.doc".encode("gbk"), Response),
+            ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
+            ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)
@@ -36,14 +34,14 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_content_type(self):
         mappings = [
-            ('text/html; charset=UTF-8', HtmlResponse),
-            ('text/xml; charset=UTF-8', XmlResponse),
-            ('application/xhtml+xml; charset=UTF-8', HtmlResponse),
-            ('application/vnd.wap.xhtml+xml; charset=utf-8', HtmlResponse),
-            ('application/xml; charset=UTF-8', XmlResponse),
-            ('application/octet-stream', Response),
-            ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse),
-            ('application/json-amazonui-streaming;charset=UTF-8', TextResponse),
+            ("text/html; charset=UTF-8", HtmlResponse),
+            ("text/xml; charset=UTF-8", XmlResponse),
+            ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
+            ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
+            ("application/xml; charset=UTF-8", XmlResponse),
+            ("application/octet-stream", Response),
+            ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse),
+            ("application/json-amazonui-streaming;charset=UTF-8", TextResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)
@@ -51,11 +49,11 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_body(self):
         mappings = [
-            (b'\x03\x02\xdf\xdd\x23', Response),
-            (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
-            (b'<html><head><title>Hello</title></head>', HtmlResponse),
+            (b"\x03\x02\xdf\xdd\x23", Response),
+            (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),
+            (b"<html><head><title>Hello</title></head>", HtmlResponse),
             # https://codersblock.com/blog/the-smallest-valid-html5-page/
-            (b'<!DOCTYPE html>\n<title>.</title>', HtmlResponse),
+            (b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
             (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
         ]
         for source, cls in mappings:
@@ -64,10 +62,21 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_headers(self):
         mappings = [
-            ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
-            ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response),
-            ({'Content-Type': ['application/octet-stream'],
-              'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
+            ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),
+            (
+                {
+                    "Content-Type": ["text/html; charset=utf-8"],
+                    "Content-Encoding": ["gzip"],
+                },
+                Response,
+            ),
+            (
+                {
+                    "Content-Type": ["application/octet-stream"],
+                    "Content-Disposition": ["attachment; filename=data.txt"],
+                },
+                TextResponse,
+            ),
         ]
         for source, cls in mappings:
             source = Headers(source)
@@ -77,14 +86,24 @@ class ResponseTypesTest(unittest.TestCase):
     def test_from_args(self):
         # TODO: add more tests that check precedence between the different arguments
         mappings = [
-            ({'url': 'http://www.example.com/data.csv'}, TextResponse),
+            ({"url": "http://www.example.com/data.csv"}, TextResponse),
             # headers takes precedence over url
-            ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}),
-              'url': 'http://www.example.com/item/'}, HtmlResponse),
-            ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}),
-              'url': 'http://www.example.com/page/'}, Response),
-
-
+            (
+                {
+                    "headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),
+                    "url": "http://www.example.com/item/",
+                },
+                HtmlResponse,
+            ),
+            (
+                {
+                    "headers": Headers(
+                        {"Content-Disposition": ['attachment; filename="data.xml.gz"']}
+                    ),
+                    "url": "http://www.example.com/page/",
+                },
+                Response,
+            ),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_args(**source)
@@ -92,7 +111,9 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_custom_mime_types_loaded(self):
         # check that mime.types files shipped with scrapy are loaded
-        self.assertEqual(responsetypes.mimetypes.guess_type('x.scrapytest')[0], 'x-scrapy/test')
+        self.assertEqual(
+            responsetypes.mimetypes.guess_type("x.scrapytest")[0], "x-scrapy/test"
+        )
 
 
 if __name__ == "__main__":
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 4b15d0fab..8d87a322a 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -34,11 +34,15 @@ class BaseRobotParserTest:
         self.parser_cls = parser_cls
 
     def test_allowed(self):
-        robotstxt_robotstxt_body = ("User-agent: * \n"
-                                    "Disallow: /disallowed \n"
-                                    "Allow: /allowed \n"
-                                    "Crawl-delay: 10".encode('utf-8'))
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        robotstxt_robotstxt_body = (
+            "User-agent: * \n"
+            "Disallow: /disallowed \n"
+            "Allow: /allowed \n"
+            "Crawl-delay: 10".encode("utf-8")
+        )
+        rp = self.parser_cls.from_crawler(
+            crawler=None, robotstxt_body=robotstxt_robotstxt_body
+        )
         self.assertTrue(rp.allowed("https://www.site.local/allowed", "*"))
         self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
 
@@ -49,35 +53,49 @@ class BaseRobotParserTest:
                                 User-agent: second
                                 Allow: /*allowed
                                 Disallow: /
-                                """.encode('utf-8')
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+                                """.encode(
+            "utf-8"
+        )
+        rp = self.parser_cls.from_crawler(
+            crawler=None, robotstxt_body=robotstxt_robotstxt_body
+        )
 
         self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first"))
-        self.assertFalse(rp.allowed("https://www.site.local/disallowed/xyz/end", "first"))
-        self.assertFalse(rp.allowed("https://www.site.local/disallowed/abc/end", "first"))
-        self.assertTrue(rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first"))
+        self.assertFalse(
+            rp.allowed("https://www.site.local/disallowed/xyz/end", "first")
+        )
+        self.assertFalse(
+            rp.allowed("https://www.site.local/disallowed/abc/end", "first")
+        )
+        self.assertTrue(
+            rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first")
+        )
 
         self.assertTrue(rp.allowed("https://www.site.local/allowed", "second"))
         self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second"))
         self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
 
     def test_length_based_precedence(self):
-        robotstxt_robotstxt_body = ("User-agent: * \n"
-                                    "Disallow: / \n"
-                                    "Allow: /page".encode('utf-8'))
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        robotstxt_robotstxt_body = (
+            "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
+        )
+        rp = self.parser_cls.from_crawler(
+            crawler=None, robotstxt_body=robotstxt_robotstxt_body
+        )
         self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
 
     def test_order_based_precedence(self):
-        robotstxt_robotstxt_body = ("User-agent: * \n"
-                                    "Disallow: / \n"
-                                    "Allow: /page".encode('utf-8'))
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        robotstxt_robotstxt_body = (
+            "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
+        )
+        rp = self.parser_cls.from_crawler(
+            crawler=None, robotstxt_body=robotstxt_robotstxt_body
+        )
         self.assertFalse(rp.allowed("https://www.site.local/page", "*"))
 
     def test_empty_response(self):
         """empty response should equal 'allow all'"""
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b'')
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"")
         self.assertTrue(rp.allowed("https://site.local/", "*"))
         self.assertTrue(rp.allowed("https://site.local/", "chrome"))
         self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
@@ -85,8 +103,10 @@ class BaseRobotParserTest:
 
     def test_garbage_response(self):
         """garbage response should be discarded, equal 'allow all'"""
-        robotstxt_robotstxt_body = b'GIF89a\xd3\x00\xfe\x00\xa2'
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"
+        rp = self.parser_cls.from_crawler(
+            crawler=None, robotstxt_body=robotstxt_robotstxt_body
+        )
         self.assertTrue(rp.allowed("https://site.local/", "*"))
         self.assertTrue(rp.allowed("https://site.local/", "chrome"))
         self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
@@ -102,25 +122,36 @@ class BaseRobotParserTest:
         Disallow: /wiki/Käyttäjä:
 
         User-Agent: UnicödeBöt
-        Disallow: /some/randome/page.html""".encode('utf-8')
-        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        Disallow: /some/randome/page.html""".encode(
+            "utf-8"
+        )
+        rp = self.parser_cls.from_crawler(
+            crawler=None, robotstxt_body=robotstxt_robotstxt_body
+        )
         self.assertTrue(rp.allowed("https://site.local/", "*"))
         self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
         self.assertFalse(rp.allowed("https://site.local/static/", "*"))
         self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
-        self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
+        self.assertFalse(
+            rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*")
+        )
         self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
         self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
-        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
+        self.assertFalse(
+            rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")
+        )
 
 
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser
+
         super()._setUp(PythonRobotParser)
 
     def test_length_based_precedence(self):
-        raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
+        raise unittest.SkipTest(
+            "RobotFileParser does not support length based directives precedence."
+        )
 
     def test_allowed_wildcards(self):
         raise unittest.SkipTest("RobotFileParser does not support wildcards.")
@@ -132,10 +163,13 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ReppyRobotParser
+
         super()._setUp(ReppyRobotParser)
 
     def test_order_based_precedence(self):
-        raise unittest.SkipTest("Reppy does not support order based directives precedence.")
+        raise unittest.SkipTest(
+            "Reppy does not support order based directives precedence."
+        )
 
 
 class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
@@ -144,10 +178,13 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import RerpRobotParser
+
         super()._setUp(RerpRobotParser)
 
     def test_length_based_precedence(self):
-        raise unittest.SkipTest("Rerp does not support length based directives precedence.")
+        raise unittest.SkipTest(
+            "Rerp does not support length based directives precedence."
+        )
 
 
 class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
@@ -156,7 +193,10 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ProtegoRobotParser
+
         super()._setUp(ProtegoRobotParser)
 
     def test_order_based_precedence(self):
-        raise unittest.SkipTest("Protego does not support order based directives precedence.")
+        raise unittest.SkipTest(
+            "Protego does not support order based directives precedence."
+        )
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 50a7755c1..67728321d 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -16,8 +16,8 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 
 
-MockEngine = collections.namedtuple('MockEngine', ['downloader'])
-MockSlot = collections.namedtuple('MockSlot', ['active'])
+MockEngine = collections.namedtuple("MockEngine", ["downloader"])
+MockSlot = collections.namedtuple("MockSlot", ["active"])
 
 
 class MockDownloader:
@@ -28,7 +28,7 @@ class MockDownloader:
         if Downloader.DOWNLOAD_SLOT in request.meta:
             return request.meta[Downloader.DOWNLOAD_SLOT]
 
-        return urlparse_cached(request).hostname or ''
+        return urlparse_cached(request).hostname or ""
 
     def increment(self, slot_key):
         slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
@@ -47,12 +47,12 @@ class MockCrawler(Crawler):
 
         settings = dict(
             SCHEDULER_DEBUG=False,
-            SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
-            SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+            SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue",
+            SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue",
             SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
             JOBDIR=jobdir,
-            DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
-            REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7',
+            DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter",
+            REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7",
         )
         super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
@@ -65,11 +65,11 @@ class SchedulerHandler:
     def create_scheduler(self):
         self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir)
         self.scheduler = Scheduler.from_crawler(self.mock_crawler)
-        self.spider = Spider(name='spider')
+        self.spider = Spider(name="spider")
         self.scheduler.open(self.spider)
 
     def close_scheduler(self):
-        self.scheduler.close('finished')
+        self.scheduler.close("finished")
         self.mock_crawler.stop()
         self.mock_crawler.engine.downloader.close()
 
@@ -80,11 +80,13 @@ class SchedulerHandler:
         self.close_scheduler()
 
 
-_PRIORITIES = [("http://foo.com/a", -2),
-               ("http://foo.com/d", 1),
-               ("http://foo.com/b", -1),
-               ("http://foo.com/c", 0),
-               ("http://foo.com/e", 2)]
+_PRIORITIES = [
+    ("http://foo.com/a", -2),
+    ("http://foo.com/d", 1),
+    ("http://foo.com/b", -1),
+    ("http://foo.com/c", 0),
+    ("http://foo.com/e", 2),
+]
 
 
 _URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"}
@@ -119,12 +121,12 @@ class BaseSchedulerInMemoryTester(SchedulerHandler):
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
-        self.assertEqual(priorities,
-                         sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+        self.assertEqual(
+            priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)
+        )
 
 
 class BaseSchedulerOnDiskTester(SchedulerHandler):
-
     def setUp(self):
         self.jobdir = tempfile.mkdtemp()
         self.create_scheduler()
@@ -172,28 +174,30 @@ class BaseSchedulerOnDiskTester(SchedulerHandler):
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
-        self.assertEqual(priorities,
-                         sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+        self.assertEqual(
+            priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)
+        )
 
 
 class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
-    priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
+    priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"
 
 
 class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
-    priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
+    priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"
 
 
-_URLS_WITH_SLOTS = [("http://foo.com/a", 'a'),
-                    ("http://foo.com/b", 'a'),
-                    ("http://foo.com/c", 'b'),
-                    ("http://foo.com/d", 'b'),
-                    ("http://foo.com/e", 'c'),
-                    ("http://foo.com/f", 'c')]
+_URLS_WITH_SLOTS = [
+    ("http://foo.com/a", "a"),
+    ("http://foo.com/b", "a"),
+    ("http://foo.com/c", "b"),
+    ("http://foo.com/d", "b"),
+    ("http://foo.com/e", "c"),
+    ("http://foo.com/f", "c"),
+]
 
 
 class TestMigration(unittest.TestCase):
-
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp()
 
@@ -202,7 +206,7 @@ class TestMigration(unittest.TestCase):
 
     def _migration(self, tmp_dir):
         prev_scheduler_handler = SchedulerHandler()
-        prev_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
+        prev_scheduler_handler.priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"
         prev_scheduler_handler.jobdir = tmp_dir
 
         prev_scheduler_handler.create_scheduler()
@@ -211,7 +215,9 @@ class TestMigration(unittest.TestCase):
         prev_scheduler_handler.close_scheduler()
 
         next_scheduler_handler = SchedulerHandler()
-        next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+        next_scheduler_handler.priority_queue_cls = (
+            "scrapy.pqueues.DownloaderAwarePriorityQueue"
+        )
         next_scheduler_handler.jobdir = tmp_dir
 
         next_scheduler_handler.create_scheduler()
@@ -239,7 +245,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots):
 
     slots_number = len(set(enqueued_slots))
     for i in range(0, len(dequeued_slots), slots_number):
-        part = dequeued_slots[i:i + slots_number]
+        part = dequeued_slots[i : i + slots_number]
         if len(part) != len(set(part)):
             return False
 
@@ -247,7 +253,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots):
 
 
 class DownloaderAwareSchedulerTestMixin:
-    priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+    priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue"
     reopen = False
 
     def test_logic(self):
@@ -276,28 +282,28 @@ class DownloaderAwareSchedulerTestMixin:
             slot = downloader._get_slot_key(request, None)
             downloader.decrement(slot)
 
-        self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS),
-                                            dequeued_slots))
+        self.assertTrue(
+            _is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), dequeued_slots)
+        )
         self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0)
 
 
-class TestSchedulerWithDownloaderAwareInMemory(DownloaderAwareSchedulerTestMixin,
-                                               BaseSchedulerInMemoryTester,
-                                               unittest.TestCase):
+class TestSchedulerWithDownloaderAwareInMemory(
+    DownloaderAwareSchedulerTestMixin, BaseSchedulerInMemoryTester, unittest.TestCase
+):
     pass
 
 
-class TestSchedulerWithDownloaderAwareOnDisk(DownloaderAwareSchedulerTestMixin,
-                                             BaseSchedulerOnDiskTester,
-                                             unittest.TestCase):
+class TestSchedulerWithDownloaderAwareOnDisk(
+    DownloaderAwareSchedulerTestMixin, BaseSchedulerOnDiskTester, unittest.TestCase
+):
     reopen = True
 
 
 class StartUrlsSpider(Spider):
-
     def __init__(self, start_urls):
         self.start_urls = start_urls
-        super().__init__(name='StartUrlsSpider')
+        super().__init__(name="StartUrlsSpider")
 
     def parse(self, response):
         pass
@@ -308,8 +314,8 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):
         self.crawler = get_crawler(
             spidercls=StartUrlsSpider,
             settings_dict={
-                'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
-                'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
+                "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue",
+                "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
             },
         )
 
@@ -324,20 +330,21 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):
             url = mockserver.url("/status?n=200", is_secure=False)
             start_urls = [url] * 6
             yield self.crawler.crawl(start_urls)
-            self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),
-                             len(start_urls))
+            self.assertEqual(
+                self.crawler.stats.get_value("downloader/response_count"),
+                len(start_urls),
+            )
 
 
 class TestIncompatibility(unittest.TestCase):
-
     def _incompatible(self):
         settings = dict(
-            SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
+            SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue",
             CONCURRENT_REQUESTS_PER_IP=1,
         )
         crawler = get_crawler(Spider, settings)
         scheduler = Scheduler.from_crawler(crawler)
-        spider = Spider(name='spider')
+        spider = Spider(name="spider")
         scheduler.open(spider)
 
     def test_incompatibility(self):
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 6602b626d..76ca777a8 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -74,7 +74,11 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin):
         self.assertIsNone(self.scheduler.open(Spider("foo")))
         self.assertIsNone(self.scheduler.close("finished"))
         self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests)
-        self.assertRaises(NotImplementedError, self.scheduler.enqueue_request, Request("https://example.org"))
+        self.assertRaises(
+            NotImplementedError,
+            self.scheduler.enqueue_request,
+            Request("https://example.org"),
+        )
         self.assertRaises(NotImplementedError, self.scheduler.next_request)
 
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index cff8d0393..ad72e068d 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -7,89 +7,104 @@ from scrapy.selector import Selector
 
 
 class SelectorTestCase(unittest.TestCase):
-
     def test_simple_selection(self):
         """Simple selector tests"""
         body = b"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
-        response = TextResponse(url="http://example.com", body=body, encoding='utf-8')
+        response = TextResponse(url="http://example.com", body=body, encoding="utf-8")
         sel = Selector(response)
 
-        xl = sel.xpath('//input')
+        xl = sel.xpath("//input")
         self.assertEqual(2, len(xl))
         for x in xl:
             assert isinstance(x, Selector)
 
         self.assertEqual(
-            sel.xpath('//input').getall(),
-            [x.get() for x in sel.xpath('//input')]
+            sel.xpath("//input").getall(), [x.get() for x in sel.xpath("//input")]
         )
         self.assertEqual(
-            [x.get() for x in sel.xpath("//input[@name='a']/@name")],
-            ['a']
+            [x.get() for x in sel.xpath("//input[@name='a']/@name")], ["a"]
         )
         self.assertEqual(
-            [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
-            ['12.0']
+            [
+                x.get()
+                for x in sel.xpath(
+                    "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))"
+                )
+            ],
+            ["12.0"],
         )
+        self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), ["xpathrules"])
         self.assertEqual(
-            sel.xpath("concat('xpath', 'rules')").getall(),
-            ['xpathrules']
-        )
-        self.assertEqual(
-            [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
-            ['12']
+            [
+                x.get()
+                for x in sel.xpath(
+                    "concat(//input[@name='a']/@value, //input[@name='b']/@value)"
+                )
+            ],
+            ["12"],
         )
 
     def test_root_base_url(self):
         body = b'<html><form action="/path"><input name="a" /></form></html>'
         url = "http://example.com"
-        response = TextResponse(url=url, body=body, encoding='utf-8')
+        response = TextResponse(url=url, body=body, encoding="utf-8")
         sel = Selector(response)
         self.assertEqual(url, sel.root.base)
 
     def test_flavor_detection(self):
         text = b'<div><img src="a.jpg"><p>Hello</div>'
-        sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
-        self.assertEqual(sel.type, 'xml')
-        self.assertEqual(sel.xpath("//div").getall(),
-                         ['<div><img src="a.jpg"><p>Hello</p></img></div>'])
+        sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8"))
+        self.assertEqual(sel.type, "xml")
+        self.assertEqual(
+            sel.xpath("//div").getall(),
+            ['<div><img src="a.jpg"><p>Hello</p></img></div>'],
+        )
 
-        sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
-        self.assertEqual(sel.type, 'html')
-        self.assertEqual(sel.xpath("//div").getall(),
-                         ['<div><img src="a.jpg"><p>Hello</p></div>'])
+        sel = Selector(HtmlResponse("http://example.com", body=text, encoding="utf-8"))
+        self.assertEqual(sel.type, "html")
+        self.assertEqual(
+            sel.xpath("//div").getall(), ['<div><img src="a.jpg"><p>Hello</p></div>']
+        )
 
     def test_http_header_encoding_precedence(self):
         # '\xa3'     = pound symbol in unicode
         # '\xc2\xa3' = pound symbol in utf-8
         # '\xa3'     = pound symbol in latin-1 (iso-8859-1)
 
-        meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
-        head = '<head>' + meta + '</head>'
+        meta = (
+            '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
+        )
+        head = "<head>" + meta + "</head>"
         body_content = '<span id="blank">\xa3</span>'
-        body = '<body>' + body_content + '</body>'
-        html = '<html>' + head + body + '</html>'
-        encoding = 'utf-8'
+        body = "<body>" + body_content + "</body>"
+        html = "<html>" + head + body + "</html>"
+        encoding = "utf-8"
         html_utf8 = html.encode(encoding)
 
-        headers = {'Content-Type': ['text/html; charset=utf-8']}
-        response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
+        headers = {"Content-Type": ["text/html; charset=utf-8"]}
+        response = HtmlResponse(
+            url="http://example.com", headers=headers, body=html_utf8
+        )
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ["\xa3"])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
-        r1 = TextResponse('http://www.example.com',
-                          body=b'<html><p>an Jos\xe9 de</p><html>',
-                          encoding='utf-8')
-        Selector(r1).xpath('//text()').getall()
+        r1 = TextResponse(
+            "http://www.example.com",
+            body=b"<html><p>an Jos\xe9 de</p><html>",
+            encoding="utf-8",
+        )
+        Selector(r1).xpath("//text()").getall()
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
-        x = Selector(text='')
+        x = Selector(text="")
         weakref.ref(x)
-        assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__"
+        assert not hasattr(
+            x, "__dict__"
+        ), f"{x.__class__.__name__} does not use __slots__"
 
     def test_selector_bad_args(self):
-        with self.assertRaisesRegex(ValueError, 'received both response and text'):
-            Selector(TextResponse(url='http://example.com', body=b''), text='')
+        with self.assertRaisesRegex(ValueError, "received both response and text"):
+            Selector(TextResponse(url="http://example.com", body=b""), text="")
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 916fe012a..9a01fd433 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,13 +1,17 @@
 import unittest
 from unittest import mock
 
-from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
-                             SETTINGS_PRIORITIES, get_settings_priority)
+from scrapy.settings import (
+    BaseSettings,
+    Settings,
+    SettingsAttribute,
+    SETTINGS_PRIORITIES,
+    get_settings_priority,
+)
 from . import default_settings
 
 
 class SettingsGlobalFuncsTest(unittest.TestCase):
-
     def test_get_settings_priority(self):
         for prio_str, prio_num in SETTINGS_PRIORITIES.items():
             self.assertEqual(get_settings_priority(prio_str), prio_num)
@@ -15,127 +19,130 @@ class SettingsGlobalFuncsTest(unittest.TestCase):
 
 
 class SettingsAttributeTest(unittest.TestCase):
-
     def setUp(self):
-        self.attribute = SettingsAttribute('value', 10)
+        self.attribute = SettingsAttribute("value", 10)
 
     def test_set_greater_priority(self):
-        self.attribute.set('value2', 20)
-        self.assertEqual(self.attribute.value, 'value2')
+        self.attribute.set("value2", 20)
+        self.assertEqual(self.attribute.value, "value2")
         self.assertEqual(self.attribute.priority, 20)
 
     def test_set_equal_priority(self):
-        self.attribute.set('value2', 10)
-        self.assertEqual(self.attribute.value, 'value2')
+        self.attribute.set("value2", 10)
+        self.assertEqual(self.attribute.value, "value2")
         self.assertEqual(self.attribute.priority, 10)
 
     def test_set_less_priority(self):
-        self.attribute.set('value2', 0)
-        self.assertEqual(self.attribute.value, 'value')
+        self.attribute.set("value2", 0)
+        self.assertEqual(self.attribute.value, "value")
         self.assertEqual(self.attribute.priority, 10)
 
     def test_overwrite_basesettings(self):
-        original_dict = {'one': 10, 'two': 20}
+        original_dict = {"one": 10, "two": 20}
         original_settings = BaseSettings(original_dict, 0)
         attribute = SettingsAttribute(original_settings, 0)
 
-        new_dict = {'three': 11, 'four': 21}
+        new_dict = {"three": 11, "four": 21}
         attribute.set(new_dict, 10)
         self.assertIsInstance(attribute.value, BaseSettings)
         self.assertCountEqual(attribute.value, new_dict)
         self.assertCountEqual(original_settings, original_dict)
 
-        new_settings = BaseSettings({'five': 12}, 0)
+        new_settings = BaseSettings({"five": 12}, 0)
         attribute.set(new_settings, 0)  # Insufficient priority
         self.assertCountEqual(attribute.value, new_dict)
         attribute.set(new_settings, 10)
         self.assertCountEqual(attribute.value, new_settings)
 
     def test_repr(self):
-        self.assertEqual(repr(self.attribute),
-                         "<SettingsAttribute value='value' priority=10>")
+        self.assertEqual(
+            repr(self.attribute), "<SettingsAttribute value='value' priority=10>"
+        )
 
 
 class BaseSettingsTest(unittest.TestCase):
-
     def setUp(self):
         self.settings = BaseSettings()
 
     def test_set_new_attribute(self):
-        self.settings.set('TEST_OPTION', 'value', 0)
-        self.assertIn('TEST_OPTION', self.settings.attributes)
+        self.settings.set("TEST_OPTION", "value", 0)
+        self.assertIn("TEST_OPTION", self.settings.attributes)
 
-        attr = self.settings.attributes['TEST_OPTION']
+        attr = self.settings.attributes["TEST_OPTION"]
         self.assertIsInstance(attr, SettingsAttribute)
-        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.value, "value")
         self.assertEqual(attr.priority, 0)
 
     def test_set_settingsattribute(self):
         myattr = SettingsAttribute(0, 30)  # Note priority 30
-        self.settings.set('TEST_ATTR', myattr, 10)
-        self.assertEqual(self.settings.get('TEST_ATTR'), 0)
-        self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
+        self.settings.set("TEST_ATTR", myattr, 10)
+        self.assertEqual(self.settings.get("TEST_ATTR"), 0)
+        self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30)
 
     def test_set_instance_identity_on_update(self):
-        attr = SettingsAttribute('value', 0)
-        self.settings.attributes = {'TEST_OPTION': attr}
-        self.settings.set('TEST_OPTION', 'othervalue', 10)
+        attr = SettingsAttribute("value", 0)
+        self.settings.attributes = {"TEST_OPTION": attr}
+        self.settings.set("TEST_OPTION", "othervalue", 10)
 
-        self.assertIn('TEST_OPTION', self.settings.attributes)
-        self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
+        self.assertIn("TEST_OPTION", self.settings.attributes)
+        self.assertIs(attr, self.settings.attributes["TEST_OPTION"])
 
     def test_set_calls_settings_attributes_methods_on_update(self):
-        attr = SettingsAttribute('value', 10)
-        with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set:
+        attr = SettingsAttribute("value", 10)
+        with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object(
+            attr, "set"
+        ) as mock_set:
 
-            self.settings.attributes = {'TEST_OPTION': attr}
+            self.settings.attributes = {"TEST_OPTION": attr}
 
             for priority in (0, 10, 20):
-                self.settings.set('TEST_OPTION', 'othervalue', priority)
-                mock_set.assert_called_once_with('othervalue', priority)
+                self.settings.set("TEST_OPTION", "othervalue", priority)
+                mock_set.assert_called_once_with("othervalue", priority)
                 self.assertFalse(mock_setattr.called)
                 mock_set.reset_mock()
                 mock_setattr.reset_mock()
 
     def test_setitem(self):
         settings = BaseSettings()
-        settings.set('key', 'a', 'default')
-        settings['key'] = 'b'
-        self.assertEqual(settings['key'], 'b')
-        self.assertEqual(settings.getpriority('key'), 20)
-        settings['key'] = 'c'
-        self.assertEqual(settings['key'], 'c')
-        settings['key2'] = 'x'
-        self.assertIn('key2', settings)
-        self.assertEqual(settings['key2'], 'x')
-        self.assertEqual(settings.getpriority('key2'), 20)
+        settings.set("key", "a", "default")
+        settings["key"] = "b"
+        self.assertEqual(settings["key"], "b")
+        self.assertEqual(settings.getpriority("key"), 20)
+        settings["key"] = "c"
+        self.assertEqual(settings["key"], "c")
+        settings["key2"] = "x"
+        self.assertIn("key2", settings)
+        self.assertEqual(settings["key2"], "x")
+        self.assertEqual(settings.getpriority("key2"), 20)
 
     def test_setdict_alias(self):
-        with mock.patch.object(self.settings, 'set') as mock_set:
-            self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
+        with mock.patch.object(self.settings, "set") as mock_set:
+            self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10)
             self.assertEqual(mock_set.call_count, 2)
-            calls = [mock.call('TEST_1', 'value1', 10),
-                     mock.call('TEST_2', 'value2', 10)]
+            calls = [
+                mock.call("TEST_1", "value1", 10),
+                mock.call("TEST_2", "value2", 10),
+            ]
             mock_set.assert_has_calls(calls, any_order=True)
 
     def test_setmodule_only_load_uppercase_vars(self):
-        class ModuleMock():
-            UPPERCASE_VAR = 'value'
-            MIXEDcase_VAR = 'othervalue'
-            lowercase_var = 'anothervalue'
+        class ModuleMock:
+            UPPERCASE_VAR = "value"
+            MIXEDcase_VAR = "othervalue"
+            lowercase_var = "anothervalue"
 
         self.settings.attributes = {}
         self.settings.setmodule(ModuleMock(), 10)
-        self.assertIn('UPPERCASE_VAR', self.settings.attributes)
-        self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
-        self.assertNotIn('lowercase_var', self.settings.attributes)
+        self.assertIn("UPPERCASE_VAR", self.settings.attributes)
+        self.assertNotIn("MIXEDcase_VAR", self.settings.attributes)
+        self.assertNotIn("lowercase_var", self.settings.attributes)
         self.assertEqual(len(self.settings.attributes), 1)
 
     def test_setmodule_alias(self):
-        with mock.patch.object(self.settings, 'set') as mock_set:
+        with mock.patch.object(self.settings, "set") as mock_set:
             self.settings.setmodule(default_settings, 10)
-            mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10)
-            mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10)
+            mock_set.assert_any_call("TEST_DEFAULT", "defvalue", 10)
+            mock_set.assert_any_call("TEST_DICT", {"key": "val"}, 10)
 
     def test_setmodule_by_path(self):
         self.settings.attributes = {}
@@ -143,11 +150,9 @@ class BaseSettingsTest(unittest.TestCase):
         ctrl_attributes = self.settings.attributes.copy()
 
         self.settings.attributes = {}
-        self.settings.setmodule(
-            'tests.test_settings.default_settings', 10)
+        self.settings.setmodule("tests.test_settings.default_settings", 10)
 
-        self.assertCountEqual(self.settings.attributes.keys(),
-                              ctrl_attributes.keys())
+        self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys())
 
         for key in ctrl_attributes.keys():
             attr = self.settings.attributes[key]
@@ -156,181 +161,196 @@ class BaseSettingsTest(unittest.TestCase):
             self.assertEqual(attr.priority, ctrl_attr.priority)
 
     def test_update(self):
-        settings = BaseSettings({'key_lowprio': 0}, priority=0)
-        settings.set('key_highprio', 10, priority=50)
-        custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11},
-                                       priority=30)
-        custom_settings.set('newkey_one', None, priority=50)
-        custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
+        settings = BaseSettings({"key_lowprio": 0}, priority=0)
+        settings.set("key_highprio", 10, priority=50)
+        custom_settings = BaseSettings(
+            {"key_lowprio": 1, "key_highprio": 11}, priority=30
+        )
+        custom_settings.set("newkey_one", None, priority=50)
+        custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None}
 
         settings.update(custom_dict, priority=20)
-        self.assertEqual(settings['key_lowprio'], 2)
-        self.assertEqual(settings.getpriority('key_lowprio'), 20)
-        self.assertEqual(settings['key_highprio'], 10)
-        self.assertIn('newkey_two', settings)
-        self.assertEqual(settings.getpriority('newkey_two'), 20)
+        self.assertEqual(settings["key_lowprio"], 2)
+        self.assertEqual(settings.getpriority("key_lowprio"), 20)
+        self.assertEqual(settings["key_highprio"], 10)
+        self.assertIn("newkey_two", settings)
+        self.assertEqual(settings.getpriority("newkey_two"), 20)
 
         settings.update(custom_settings)
-        self.assertEqual(settings['key_lowprio'], 1)
-        self.assertEqual(settings.getpriority('key_lowprio'), 30)
-        self.assertEqual(settings['key_highprio'], 10)
-        self.assertIn('newkey_one', settings)
-        self.assertEqual(settings.getpriority('newkey_one'), 50)
+        self.assertEqual(settings["key_lowprio"], 1)
+        self.assertEqual(settings.getpriority("key_lowprio"), 30)
+        self.assertEqual(settings["key_highprio"], 10)
+        self.assertIn("newkey_one", settings)
+        self.assertEqual(settings.getpriority("newkey_one"), 50)
 
-        settings.update({'key_lowprio': 3}, priority=20)
-        self.assertEqual(settings['key_lowprio'], 1)
+        settings.update({"key_lowprio": 3}, priority=20)
+        self.assertEqual(settings["key_lowprio"], 1)
 
     def test_update_jsonstring(self):
-        settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
+        settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})})
         settings.update('{"number": 1, "newnumber": 2}')
-        self.assertEqual(settings['number'], 1)
-        self.assertEqual(settings['newnumber'], 2)
+        self.assertEqual(settings["number"], 1)
+        self.assertEqual(settings["newnumber"], 2)
         settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
-        self.assertEqual(settings['dict']['key'], "newval")
-        self.assertEqual(settings['dict']['newkey'], "newval2")
+        self.assertEqual(settings["dict"]["key"], "newval")
+        self.assertEqual(settings["dict"]["newkey"], "newval2")
 
     def test_delete(self):
-        settings = BaseSettings({'key': None})
-        settings.set('key_highprio', None, priority=50)
-        settings.delete('key')
-        settings.delete('key_highprio')
-        self.assertNotIn('key', settings)
-        self.assertIn('key_highprio', settings)
-        del settings['key_highprio']
-        self.assertNotIn('key_highprio', settings)
+        settings = BaseSettings({"key": None})
+        settings.set("key_highprio", None, priority=50)
+        settings.delete("key")
+        settings.delete("key_highprio")
+        self.assertNotIn("key", settings)
+        self.assertIn("key_highprio", settings)
+        del settings["key_highprio"]
+        self.assertNotIn("key_highprio", settings)
 
     def test_get(self):
         test_configuration = {
-            'TEST_ENABLED1': '1',
-            'TEST_ENABLED2': True,
-            'TEST_ENABLED3': 1,
-            'TEST_ENABLED4': 'True',
-            'TEST_ENABLED5': 'true',
-            'TEST_ENABLED_WRONG': 'on',
-            'TEST_DISABLED1': '0',
-            'TEST_DISABLED2': False,
-            'TEST_DISABLED3': 0,
-            'TEST_DISABLED4': 'False',
-            'TEST_DISABLED5': 'false',
-            'TEST_DISABLED_WRONG': 'off',
-            'TEST_INT1': 123,
-            'TEST_INT2': '123',
-            'TEST_FLOAT1': 123.45,
-            'TEST_FLOAT2': '123.45',
-            'TEST_LIST1': ['one', 'two'],
-            'TEST_LIST2': 'one,two',
-            'TEST_STR': 'value',
-            'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
-            'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
+            "TEST_ENABLED1": "1",
+            "TEST_ENABLED2": True,
+            "TEST_ENABLED3": 1,
+            "TEST_ENABLED4": "True",
+            "TEST_ENABLED5": "true",
+            "TEST_ENABLED_WRONG": "on",
+            "TEST_DISABLED1": "0",
+            "TEST_DISABLED2": False,
+            "TEST_DISABLED3": 0,
+            "TEST_DISABLED4": "False",
+            "TEST_DISABLED5": "false",
+            "TEST_DISABLED_WRONG": "off",
+            "TEST_INT1": 123,
+            "TEST_INT2": "123",
+            "TEST_FLOAT1": 123.45,
+            "TEST_FLOAT2": "123.45",
+            "TEST_LIST1": ["one", "two"],
+            "TEST_LIST2": "one,two",
+            "TEST_STR": "value",
+            "TEST_DICT1": {"key1": "val1", "ke2": 3},
+            "TEST_DICT2": '{"key1": "val1", "ke2": 3}',
         }
         settings = self.settings
-        settings.attributes = {key: SettingsAttribute(value, 0) for key, value
-                               in test_configuration.items()}
+        settings.attributes = {
+            key: SettingsAttribute(value, 0)
+            for key, value in test_configuration.items()
+        }
 
-        self.assertTrue(settings.getbool('TEST_ENABLED1'))
-        self.assertTrue(settings.getbool('TEST_ENABLED2'))
-        self.assertTrue(settings.getbool('TEST_ENABLED3'))
-        self.assertTrue(settings.getbool('TEST_ENABLED4'))
-        self.assertTrue(settings.getbool('TEST_ENABLED5'))
-        self.assertFalse(settings.getbool('TEST_ENABLEDx'))
-        self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
-        self.assertFalse(settings.getbool('TEST_DISABLED1'))
-        self.assertFalse(settings.getbool('TEST_DISABLED2'))
-        self.assertFalse(settings.getbool('TEST_DISABLED3'))
-        self.assertFalse(settings.getbool('TEST_DISABLED4'))
-        self.assertFalse(settings.getbool('TEST_DISABLED5'))
-        self.assertEqual(settings.getint('TEST_INT1'), 123)
-        self.assertEqual(settings.getint('TEST_INT2'), 123)
-        self.assertEqual(settings.getint('TEST_INTx'), 0)
-        self.assertEqual(settings.getint('TEST_INTx', 45), 45)
-        self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
-        self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
-        self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
-        self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
-        self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
-        self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
-        self.assertEqual(settings.getlist('TEST_LISTx'), [])
-        self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
-        self.assertEqual(settings['TEST_STR'], 'value')
-        self.assertEqual(settings.get('TEST_STR'), 'value')
-        self.assertEqual(settings['TEST_STRx'], None)
-        self.assertEqual(settings.get('TEST_STRx'), None)
-        self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
-        self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
-        self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
-        self.assertEqual(settings.getdict('TEST_DICT3'), {})
-        self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
-        self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
-        self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG')
-        self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG')
+        self.assertTrue(settings.getbool("TEST_ENABLED1"))
+        self.assertTrue(settings.getbool("TEST_ENABLED2"))
+        self.assertTrue(settings.getbool("TEST_ENABLED3"))
+        self.assertTrue(settings.getbool("TEST_ENABLED4"))
+        self.assertTrue(settings.getbool("TEST_ENABLED5"))
+        self.assertFalse(settings.getbool("TEST_ENABLEDx"))
+        self.assertTrue(settings.getbool("TEST_ENABLEDx", True))
+        self.assertFalse(settings.getbool("TEST_DISABLED1"))
+        self.assertFalse(settings.getbool("TEST_DISABLED2"))
+        self.assertFalse(settings.getbool("TEST_DISABLED3"))
+        self.assertFalse(settings.getbool("TEST_DISABLED4"))
+        self.assertFalse(settings.getbool("TEST_DISABLED5"))
+        self.assertEqual(settings.getint("TEST_INT1"), 123)
+        self.assertEqual(settings.getint("TEST_INT2"), 123)
+        self.assertEqual(settings.getint("TEST_INTx"), 0)
+        self.assertEqual(settings.getint("TEST_INTx", 45), 45)
+        self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45)
+        self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45)
+        self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0)
+        self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0)
+        self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"])
+        self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"])
+        self.assertEqual(settings.getlist("TEST_LISTx"), [])
+        self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"])
+        self.assertEqual(settings["TEST_STR"], "value")
+        self.assertEqual(settings.get("TEST_STR"), "value")
+        self.assertEqual(settings["TEST_STRx"], None)
+        self.assertEqual(settings.get("TEST_STRx"), None)
+        self.assertEqual(settings.get("TEST_STRx", "default"), "default")
+        self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3})
+        self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3})
+        self.assertEqual(settings.getdict("TEST_DICT3"), {})
+        self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5})
+        self.assertRaises(ValueError, settings.getdict, "TEST_LIST1")
+        self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG")
+        self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG")
 
     def test_getpriority(self):
-        settings = BaseSettings({'key': 'value'}, priority=99)
-        self.assertEqual(settings.getpriority('key'), 99)
-        self.assertEqual(settings.getpriority('nonexistentkey'), None)
+        settings = BaseSettings({"key": "value"}, priority=99)
+        self.assertEqual(settings.getpriority("key"), 99)
+        self.assertEqual(settings.getpriority("nonexistentkey"), None)
 
     def test_getwithbase(self):
-        s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
-                          'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
-                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
-        s['TEST'].set(2, 200, 'cmdline')
-        self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
-        self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
-        self.assertEqual(s.getwithbase('NONEXISTENT'), {})
+        s = BaseSettings(
+            {
+                "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),
+                "TEST": BaseSettings({1: 10, 3: 30}, "default"),
+                "HASNOBASE": BaseSettings({3: 3000}, "default"),
+            }
+        )
+        s["TEST"].set(2, 200, "cmdline")
+        self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30})
+        self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"])
+        self.assertEqual(s.getwithbase("NONEXISTENT"), {})
 
     def test_maxpriority(self):
         # Empty settings should return 'default'
         self.assertEqual(self.settings.maxpriority(), 0)
-        self.settings.set('A', 0, 10)
-        self.settings.set('B', 0, 30)
+        self.settings.set("A", 0, 10)
+        self.settings.set("B", 0, 30)
         self.assertEqual(self.settings.maxpriority(), 30)
 
     def test_copy(self):
         values = {
-            'TEST_BOOL': True,
-            'TEST_LIST': ['one', 'two'],
-            'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
-                                   ['second_one', 'second_two']]
+            "TEST_BOOL": True,
+            "TEST_LIST": ["one", "two"],
+            "TEST_LIST_OF_LISTS": [
+                ["first_one", "first_two"],
+                ["second_one", "second_two"],
+            ],
         }
         self.settings.setdict(values)
         copy = self.settings.copy()
-        self.settings.set('TEST_BOOL', False)
-        self.assertTrue(copy.get('TEST_BOOL'))
+        self.settings.set("TEST_BOOL", False)
+        self.assertTrue(copy.get("TEST_BOOL"))
 
-        test_list = self.settings.get('TEST_LIST')
-        test_list.append('three')
-        self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
+        test_list = self.settings.get("TEST_LIST")
+        test_list.append("three")
+        self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"])
 
-        test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
-        test_list_of_lists[0].append('first_three')
-        self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
-                             ['first_one', 'first_two'])
+        test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS")
+        test_list_of_lists[0].append("first_three")
+        self.assertListEqual(
+            copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"]
+        )
 
     def test_copy_to_dict(self):
-        s = BaseSettings({'TEST_STRING': 'a string',
-                          'TEST_LIST': [1, 2],
-                          'TEST_BOOLEAN': False,
-                          'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
-                          'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
-                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
+        s = BaseSettings(
+            {
+                "TEST_STRING": "a string",
+                "TEST_LIST": [1, 2],
+                "TEST_BOOLEAN": False,
+                "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),
+                "TEST": BaseSettings({1: 10, 3: 30}, "default"),
+                "HASNOBASE": BaseSettings({3: 3000}, "default"),
+            }
+        )
         self.assertDictEqual(
             s.copy_to_dict(),
             {
-                'HASNOBASE': {3: 3000},
-                'TEST': {1: 10, 3: 30},
-                'TEST_BASE': {1: 1, 2: 2},
-                'TEST_LIST': [1, 2],
-                'TEST_BOOLEAN': False,
-                'TEST_STRING': 'a string',
-            }
+                "HASNOBASE": {3: 3000},
+                "TEST": {1: 10, 3: 30},
+                "TEST_BASE": {1: 1, 2: 2},
+                "TEST_LIST": [1, 2],
+                "TEST_BOOLEAN": False,
+                "TEST_STRING": "a string",
+            },
         )
 
     def test_freeze(self):
         self.settings.freeze()
         with self.assertRaises(TypeError) as cm:
-            self.settings.set('TEST_BOOL', False)
-            self.assertEqual(str(cm.exception),
-                             "Trying to modify an immutable Settings object")
+            self.settings.set("TEST_BOOL", False)
+            self.assertEqual(
+                str(cm.exception), "Trying to modify an immutable Settings object"
+            )
 
     def test_frozencopy(self):
         frozencopy = self.settings.frozencopy()
@@ -339,83 +359,84 @@ class BaseSettingsTest(unittest.TestCase):
 
 
 class SettingsTest(unittest.TestCase):
-
     def setUp(self):
         self.settings = Settings()
 
-    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
-    @mock.patch('scrapy.settings.default_settings', default_settings)
+    @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10})
+    @mock.patch("scrapy.settings.default_settings", default_settings)
     def test_initial_defaults(self):
         settings = Settings()
         self.assertEqual(len(settings.attributes), 2)
-        self.assertIn('TEST_DEFAULT', settings.attributes)
+        self.assertIn("TEST_DEFAULT", settings.attributes)
 
-        attr = settings.attributes['TEST_DEFAULT']
+        attr = settings.attributes["TEST_DEFAULT"]
         self.assertIsInstance(attr, SettingsAttribute)
-        self.assertEqual(attr.value, 'defvalue')
+        self.assertEqual(attr.value, "defvalue")
         self.assertEqual(attr.priority, 10)
 
-    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
-    @mock.patch('scrapy.settings.default_settings', {})
+    @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {})
+    @mock.patch("scrapy.settings.default_settings", {})
     def test_initial_values(self):
-        settings = Settings({'TEST_OPTION': 'value'}, 10)
+        settings = Settings({"TEST_OPTION": "value"}, 10)
         self.assertEqual(len(settings.attributes), 1)
-        self.assertIn('TEST_OPTION', settings.attributes)
+        self.assertIn("TEST_OPTION", settings.attributes)
 
-        attr = settings.attributes['TEST_OPTION']
+        attr = settings.attributes["TEST_OPTION"]
         self.assertIsInstance(attr, SettingsAttribute)
-        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.value, "value")
         self.assertEqual(attr.priority, 10)
 
-    @mock.patch('scrapy.settings.default_settings', default_settings)
+    @mock.patch("scrapy.settings.default_settings", default_settings)
     def test_autopromote_dicts(self):
         settings = Settings()
-        mydict = settings.get('TEST_DICT')
+        mydict = settings.get("TEST_DICT")
         self.assertIsInstance(mydict, BaseSettings)
-        self.assertIn('key', mydict)
-        self.assertEqual(mydict['key'], 'val')
-        self.assertEqual(mydict.getpriority('key'), 0)
+        self.assertIn("key", mydict)
+        self.assertEqual(mydict["key"], "val")
+        self.assertEqual(mydict.getpriority("key"), 0)
 
-    @mock.patch('scrapy.settings.default_settings', default_settings)
+    @mock.patch("scrapy.settings.default_settings", default_settings)
     def test_getdict_autodegrade_basesettings(self):
         settings = Settings()
-        mydict = settings.getdict('TEST_DICT')
+        mydict = settings.getdict("TEST_DICT")
         self.assertIsInstance(mydict, dict)
         self.assertEqual(len(mydict), 1)
-        self.assertIn('key', mydict)
-        self.assertEqual(mydict['key'], 'val')
+        self.assertIn("key", mydict)
+        self.assertEqual(mydict["key"], "val")
 
     def test_passing_objects_as_values(self):
         from scrapy.core.downloader.handlers.file import FileDownloadHandler
         from scrapy.utils.misc import create_instance
         from scrapy.utils.test import get_crawler
 
-        class TestPipeline():
+        class TestPipeline:
             def process_item(self, i, s):
                 return i
 
-        settings = Settings({
-            'ITEM_PIPELINES': {
-                TestPipeline: 800,
-            },
-            'DOWNLOAD_HANDLERS': {
-                'ftp': FileDownloadHandler,
-            },
-        })
+        settings = Settings(
+            {
+                "ITEM_PIPELINES": {
+                    TestPipeline: 800,
+                },
+                "DOWNLOAD_HANDLERS": {
+                    "ftp": FileDownloadHandler,
+                },
+            }
+        )
 
-        self.assertIn('ITEM_PIPELINES', settings.attributes)
+        self.assertIn("ITEM_PIPELINES", settings.attributes)
 
-        mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem()
+        mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem()
         self.assertEqual(priority, 800)
         self.assertEqual(mypipeline, TestPipeline)
         self.assertIsInstance(mypipeline(), TestPipeline)
-        self.assertEqual(mypipeline().process_item('item', None), 'item')
+        self.assertEqual(mypipeline().process_item("item", None), "item")
 
-        myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp')
+        myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
         self.assertEqual(myhandler, FileDownloadHandler)
         myhandler_instance = create_instance(myhandler, None, get_crawler())
         self.assertIsInstance(myhandler_instance, FileDownloadHandler)
-        self.assertTrue(hasattr(myhandler_instance, 'download_request'))
+        self.assertTrue(hasattr(myhandler_instance, "download_request"))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_settings/default_settings.py b/tests/test_settings/default_settings.py
index 26a555275..2a2142b32 100644
--- a/tests/test_settings/default_settings.py
+++ b/tests/test_settings/default_settings.py
@@ -1,4 +1,3 @@
+TEST_DEFAULT = "defvalue"
 
-TEST_DEFAULT = 'defvalue'
-
-TEST_DICT = {'key': 'val'}
+TEST_DICT = {"key": "val"}
diff --git a/tests/test_signals.py b/tests/test_signals.py
index a43f00b27..4c6ffabdc 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -9,15 +9,16 @@ from tests.mockserver import MockServer
 
 
 class ItemSpider(Spider):
-    name = 'itemspider'
+    name = "itemspider"
 
     def start_requests(self):
         for index in range(10):
-            yield Request(self.mockserver.url(f'/status?n=200&id={index}'),
-                          meta={'index': index})
+            yield Request(
+                self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index}
+            )
 
     def parse(self, response):
-        return {'index': response.meta['index']}
+        return {"index": response.meta["index"]}
 
 
 class AsyncSignalTestCase(unittest.TestCase):
@@ -41,4 +42,4 @@ class AsyncSignalTestCase(unittest.TestCase):
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 10)
         for index in range(10):
-            self.assertIn({'index': index}, self.items)
+            self.assertIn({"index": index}, self.items)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index cb66066b0..540091516 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -37,39 +37,39 @@ class SpiderTest(unittest.TestCase):
 
     def test_base_spider(self):
         spider = self.spider_class("example.com")
-        self.assertEqual(spider.name, 'example.com')
+        self.assertEqual(spider.name, "example.com")
         self.assertEqual(spider.start_urls, [])
 
     def test_start_requests(self):
-        spider = self.spider_class('example.com')
+        spider = self.spider_class("example.com")
         start_requests = spider.start_requests()
         self.assertTrue(inspect.isgenerator(start_requests))
         self.assertEqual(list(start_requests), [])
 
     def test_spider_args(self):
         """``__init__`` method arguments are assigned to spider attributes"""
-        spider = self.spider_class('example.com', foo='bar')
-        self.assertEqual(spider.foo, 'bar')
+        spider = self.spider_class("example.com", foo="bar")
+        self.assertEqual(spider.foo, "bar")
 
     def test_spider_without_name(self):
         """``__init__`` method arguments are assigned to spider attributes"""
         self.assertRaises(ValueError, self.spider_class)
-        self.assertRaises(ValueError, self.spider_class, somearg='foo')
+        self.assertRaises(ValueError, self.spider_class, somearg="foo")
 
     def test_from_crawler_crawler_and_settings_population(self):
         crawler = get_crawler()
-        spider = self.spider_class.from_crawler(crawler, 'example.com')
-        self.assertTrue(hasattr(spider, 'crawler'))
+        spider = self.spider_class.from_crawler(crawler, "example.com")
+        self.assertTrue(hasattr(spider, "crawler"))
         self.assertIs(spider.crawler, crawler)
-        self.assertTrue(hasattr(spider, 'settings'))
+        self.assertTrue(hasattr(spider, "settings"))
         self.assertIs(spider.settings, crawler.settings)
 
     def test_from_crawler_init_call(self):
-        with mock.patch.object(self.spider_class, '__init__',
-                               return_value=None) as mock_init:
-            self.spider_class.from_crawler(get_crawler(), 'example.com',
-                                           foo='bar')
-            mock_init.assert_called_once_with('example.com', foo='bar')
+        with mock.patch.object(
+            self.spider_class, "__init__", return_value=None
+        ) as mock_init:
+            self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
+            mock_init.assert_called_once_with("example.com", foo="bar")
 
     def test_closed_signal_call(self):
         class TestSpider(self.spider_class):
@@ -79,39 +79,39 @@ class SpiderTest(unittest.TestCase):
                 self.closed_called = True
 
         crawler = get_crawler()
-        spider = TestSpider.from_crawler(crawler, 'example.com')
-        crawler.signals.send_catch_log(signal=signals.spider_opened,
-                                       spider=spider)
-        crawler.signals.send_catch_log(signal=signals.spider_closed,
-                                       spider=spider, reason=None)
+        spider = TestSpider.from_crawler(crawler, "example.com")
+        crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
+        crawler.signals.send_catch_log(
+            signal=signals.spider_closed, spider=spider, reason=None
+        )
         self.assertTrue(spider.closed_called)
 
     def test_update_settings(self):
-        spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
-        project_settings = {'TEST1': 'project', 'TEST3': 'project'}
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {"TEST1": "project", "TEST3": "project"}
         self.spider_class.custom_settings = spider_settings
-        settings = Settings(project_settings, priority='project')
+        settings = Settings(project_settings, priority="project")
 
         self.spider_class.update_settings(settings)
-        self.assertEqual(settings.get('TEST1'), 'spider')
-        self.assertEqual(settings.get('TEST2'), 'spider')
-        self.assertEqual(settings.get('TEST3'), 'project')
+        self.assertEqual(settings.get("TEST1"), "spider")
+        self.assertEqual(settings.get("TEST2"), "spider")
+        self.assertEqual(settings.get("TEST3"), "project")
 
     def test_logger(self):
-        spider = self.spider_class('example.com')
+        spider = self.spider_class("example.com")
         with LogCapture() as lc:
-            spider.logger.info('test log msg')
-        lc.check(('example.com', 'INFO', 'test log msg'))
+            spider.logger.info("test log msg")
+        lc.check(("example.com", "INFO", "test log msg"))
 
         record = lc.records[0]
-        self.assertIn('spider', record.__dict__)
+        self.assertIn("spider", record.__dict__)
         self.assertIs(record.spider, spider)
 
     def test_log(self):
-        spider = self.spider_class('example.com')
-        with mock.patch('scrapy.spiders.Spider.logger') as mock_logger:
-            spider.log('test log msg', 'INFO')
-        mock_logger.log.assert_called_once_with('INFO', 'test log msg')
+        spider = self.spider_class("example.com")
+        with mock.patch("scrapy.spiders.Spider.logger") as mock_logger:
+            spider.log("test log msg", "INFO")
+        mock_logger.log.assert_called_once_with("INFO", "test log msg")
 
 
 class InitSpiderTest(SpiderTest):
@@ -132,37 +132,45 @@ class XMLFeedSpiderTest(SpiderTest):
         </url>
         <url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url>
         </urlset>"""
-        response = XmlResponse(url='http://example.com/sitemap.xml', body=body)
+        response = XmlResponse(url="http://example.com/sitemap.xml", body=body)
 
         class _XMLSpider(self.spider_class):
-            itertag = 'url'
+            itertag = "url"
             namespaces = (
-                ('a', 'http://www.google.com/schemas/sitemap/0.84'),
-                ('b', 'http://www.example.com/schemas/extras/1.0'),
+                ("a", "http://www.google.com/schemas/sitemap/0.84"),
+                ("b", "http://www.example.com/schemas/extras/1.0"),
             )
 
             def parse_node(self, response, selector):
                 yield {
-                    'loc': selector.xpath('a:loc/text()').getall(),
-                    'updated': selector.xpath('b:updated/text()').getall(),
-                    'other': selector.xpath('other/@value').getall(),
-                    'custom': selector.xpath('other/@b:custom').getall(),
+                    "loc": selector.xpath("a:loc/text()").getall(),
+                    "updated": selector.xpath("b:updated/text()").getall(),
+                    "other": selector.xpath("other/@value").getall(),
+                    "custom": selector.xpath("other/@b:custom").getall(),
                 }
 
-        for iterator in ('iternodes', 'xml'):
-            spider = _XMLSpider('example', iterator=iterator)
+        for iterator in ("iternodes", "xml"):
+            spider = _XMLSpider("example", iterator=iterator)
             output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
-            self.assertEqual(output, [
-                {'loc': ['http://www.example.com/Special-Offers.html'],
-                 'updated': ['2009-08-16'],
-                 'custom': ['fuu'],
-                 'other': ['bar']},
-                {'loc': [],
-                 'updated': ['2009-08-16'],
-                 'other': ['foo'],
-                 'custom': []},
-            ], iterator)
+            self.assertEqual(
+                output,
+                [
+                    {
+                        "loc": ["http://www.example.com/Special-Offers.html"],
+                        "updated": ["2009-08-16"],
+                        "custom": ["fuu"],
+                        "other": ["bar"],
+                    },
+                    {
+                        "loc": [],
+                        "updated": ["2009-08-16"],
+                        "other": ["foo"],
+                        "custom": [],
+                    },
+                ],
+                iterator,
+            )
 
 
 class CSVFeedSpiderTest(SpiderTest):
@@ -170,7 +178,7 @@ class CSVFeedSpiderTest(SpiderTest):
     spider_class = CSVFeedSpider
 
     def test_parse_rows(self):
-        body = get_testdata('feeds', 'feed-sample6.csv')
+        body = get_testdata("feeds", "feed-sample6.csv")
         response = Response("http://example.org/dummy.csv", body=body)
 
         class _CrawlSpider(self.spider_class):
@@ -183,7 +191,7 @@ class CSVFeedSpiderTest(SpiderTest):
 
         spider = _CrawlSpider()
         rows = list(spider.parse_rows(response))
-        assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'}
+        assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"}
         assert len(rows) == 4
 
 
@@ -203,34 +211,38 @@ class CrawlSpiderTest(SpiderTest):
 
     def test_rule_without_link_extractor(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
-            rules = (
-                Rule(),
-            )
+            allowed_domains = ["example.org"]
+            rules = (Rule(),)
 
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.org/somepage/item/12.html',
-                          'http://example.org/about.html',
-                          'http://example.org/nofollow.html'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.org/somepage/item/12.html",
+                "http://example.org/about.html",
+                "http://example.org/nofollow.html",
+            ],
+        )
 
     def test_process_links(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
-            rules = (
-                Rule(LinkExtractor(), process_links="dummy_process_links"),
-            )
+            allowed_domains = ["example.org"]
+            rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
 
             def dummy_process_links(self, links):
                 return links
@@ -239,47 +251,54 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.org/somepage/item/12.html',
-                          'http://example.org/about.html',
-                          'http://example.org/nofollow.html'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.org/somepage/item/12.html",
+                "http://example.org/about.html",
+                "http://example.org/nofollow.html",
+            ],
+        )
 
     def test_process_links_filter(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         class _CrawlSpider(self.spider_class):
             import re
 
             name = "test"
-            allowed_domains = ['example.org']
-            rules = (
-                Rule(LinkExtractor(), process_links="filter_process_links"),
-            )
-            _test_regex = re.compile('nofollow')
+            allowed_domains = ["example.org"]
+            rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
+            _test_regex = re.compile("nofollow")
 
             def filter_process_links(self, links):
-                return [link for link in links
-                        if not self._test_regex.search(link.url)]
+                return [link for link in links if not self._test_regex.search(link.url)]
 
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 2)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.org/somepage/item/12.html',
-                          'http://example.org/about.html'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.org/somepage/item/12.html",
+                "http://example.org/about.html",
+            ],
+        )
 
     def test_process_links_generator(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
-            rules = (
-                Rule(LinkExtractor(), process_links="dummy_process_links"),
-            )
+            allowed_domains = ["example.org"]
+            rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
 
             def dummy_process_links(self, links):
                 for link in links:
@@ -289,21 +308,27 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.org/somepage/item/12.html',
-                          'http://example.org/about.html',
-                          'http://example.org/nofollow.html'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.org/somepage/item/12.html",
+                "http://example.org/about.html",
+                "http://example.org/nofollow.html",
+            ],
+        )
 
     def test_process_request(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         def process_request_change_domain(request, response):
-            return request.replace(url=request.url.replace('.org', '.com'))
+            return request.replace(url=request.url.replace(".org", ".com"))
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
+            allowed_domains = ["example.org"]
             rules = (
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
@@ -312,47 +337,61 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.com/somepage/item/12.html',
-                          'http://example.com/about.html',
-                          'http://example.com/nofollow.html'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.com/somepage/item/12.html",
+                "http://example.com/about.html",
+                "http://example.com/nofollow.html",
+            ],
+        )
 
     def test_process_request_with_response(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         def process_request_meta_response_class(request, response):
-            request.meta['response_class'] = response.__class__.__name__
+            request.meta["response_class"] = response.__class__.__name__
             return request
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
+            allowed_domains = ["example.org"]
             rules = (
-                Rule(LinkExtractor(), process_request=process_request_meta_response_class),
+                Rule(
+                    LinkExtractor(), process_request=process_request_meta_response_class
+                ),
             )
 
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.org/somepage/item/12.html',
-                          'http://example.org/about.html',
-                          'http://example.org/nofollow.html'])
-        self.assertEqual([r.meta['response_class'] for r in output],
-                         ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.org/somepage/item/12.html",
+                "http://example.org/about.html",
+                "http://example.org/nofollow.html",
+            ],
+        )
+        self.assertEqual(
+            [r.meta["response_class"] for r in output],
+            ["HtmlResponse", "HtmlResponse", "HtmlResponse"],
+        )
 
     def test_process_request_instance_method(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
-            rules = (
-                Rule(LinkExtractor(), process_request='process_request_upper'),
-            )
+            allowed_domains = ["example.org"]
+            rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
 
             def process_request_upper(self, request, response):
                 return request.replace(url=request.url.upper())
@@ -361,55 +400,69 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'),
-                          safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'),
-                          safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
+                safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
+                safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
+            ],
+        )
 
     def test_process_request_instance_method_with_response(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
 
         class _CrawlSpider(self.spider_class):
             name = "test"
-            allowed_domains = ['example.org']
+            allowed_domains = ["example.org"]
             rules = (
-                Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
+                Rule(
+                    LinkExtractor(),
+                    process_request="process_request_meta_response_class",
+                ),
             )
 
             def process_request_meta_response_class(self, request, response):
-                request.meta['response_class'] = response.__class__.__name__
+                request.meta["response_class"] = response.__class__.__name__
                 return request
 
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.org/somepage/item/12.html',
-                          'http://example.org/about.html',
-                          'http://example.org/nofollow.html'])
-        self.assertEqual([r.meta['response_class'] for r in output],
-                         ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+        self.assertEqual(
+            [r.url for r in output],
+            [
+                "http://example.org/somepage/item/12.html",
+                "http://example.org/about.html",
+                "http://example.org/nofollow.html",
+            ],
+        )
+        self.assertEqual(
+            [r.meta["response_class"] for r in output],
+            ["HtmlResponse", "HtmlResponse", "HtmlResponse"],
+        )
 
     def test_follow_links_attribute_population(self):
         crawler = get_crawler()
-        spider = self.spider_class.from_crawler(crawler, 'example.com')
-        self.assertTrue(hasattr(spider, '_follow_links'))
+        spider = self.spider_class.from_crawler(crawler, "example.com")
+        self.assertTrue(hasattr(spider, "_follow_links"))
         self.assertTrue(spider._follow_links)
 
-        settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
+        settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
         crawler = get_crawler(settings_dict=settings_dict)
-        spider = self.spider_class.from_crawler(crawler, 'example.com')
-        self.assertTrue(hasattr(spider, '_follow_links'))
+        spider = self.spider_class.from_crawler(crawler, "example.com")
+        self.assertTrue(hasattr(spider, "_follow_links"))
         self.assertFalse(spider._follow_links)
 
     def test_start_url(self):
         spider = self.spider_class("example.com")
-        spider.start_url = 'https://www.example.com'
+        spider.start_url = "https://www.example.com"
 
-        with self.assertRaisesRegex(AttributeError,
-                                    r'^Crawling could not start.*$'):
+        with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"):
             list(spider.start_requests())
 
 
@@ -419,7 +472,7 @@ class SitemapSpiderTest(SpiderTest):
 
     BODY = b"SITEMAP"
     f = BytesIO()
-    g = gzip.GzipFile(fileobj=f, mode='w+b')
+    g = gzip.GzipFile(fileobj=f, mode="w+b")
     g.write(BODY)
     g.close()
     GZBODY = f.getvalue()
@@ -439,8 +492,11 @@ class SitemapSpiderTest(SpiderTest):
         self.assertSitemapBody(r, None)
 
     def test_get_sitemap_body_gzip_headers(self):
-        r = Response(url="http://www.example.com/sitemap", body=self.GZBODY,
-                     headers={"content-type": "application/gzip"})
+        r = Response(
+            url="http://www.example.com/sitemap",
+            body=self.GZBODY,
+            headers={"content-type": "application/gzip"},
+        )
         self.assertSitemapBody(r, self.BODY)
 
     def test_get_sitemap_body_xml_url(self):
@@ -465,11 +521,15 @@ Sitemap: /sitemap-relative-url.xml
 
         r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
         spider = self.spider_class("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://example.com/sitemap.xml',
-                          'http://example.com/sitemap-product-index.xml',
-                          'http://example.com/sitemap-uppercase.xml',
-                          'http://www.example.com/sitemap-relative-url.xml'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            [
+                "http://example.com/sitemap.xml",
+                "http://example.com/sitemap-product-index.xml",
+                "http://example.com/sitemap-uppercase.xml",
+                "http://www.example.com/sitemap-relative-url.xml",
+            ],
+        )
 
     def test_alternate_url_locs(self):
         sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -488,15 +548,21 @@ Sitemap: /sitemap-relative-url.xml
     </urlset>"""
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
         spider = self.spider_class("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/english/'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            ["http://www.example.com/english/"],
+        )
 
         spider.sitemap_alternate_links = True
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/english/',
-                          'http://www.example.com/deutsch/',
-                          'http://www.example.com/schweiz-deutsch/',
-                          'http://www.example.com/italiano/'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            [
+                "http://www.example.com/english/",
+                "http://www.example.com/deutsch/",
+                "http://www.example.com/schweiz-deutsch/",
+                "http://www.example.com/italiano/",
+            ],
+        )
 
     def test_sitemap_filter(self):
         sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -515,20 +581,24 @@ Sitemap: /sitemap-relative-url.xml
         class FilteredSitemapSpider(self.spider_class):
             def sitemap_filter(self, entries):
                 from datetime import datetime
+
                 for entry in entries:
-                    date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
+                    date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
                     if date_time.year > 2008:
                         yield entry
 
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
         spider = self.spider_class("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/english/',
-                          'http://www.example.com/portuguese/'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            ["http://www.example.com/english/", "http://www.example.com/portuguese/"],
+        )
 
         spider = FilteredSitemapSpider("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/english/'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            ["http://www.example.com/english/"],
+        )
 
     def test_sitemap_filter_with_alternate_links(self):
         sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -549,21 +619,27 @@ Sitemap: /sitemap-relative-url.xml
         class FilteredSitemapSpider(self.spider_class):
             def sitemap_filter(self, entries):
                 for entry in entries:
-                    alternate_links = entry.get('alternate', tuple())
+                    alternate_links = entry.get("alternate", tuple())
                     for link in alternate_links:
-                        if '/deutsch/' in link:
-                            entry['loc'] = link
+                        if "/deutsch/" in link:
+                            entry["loc"] = link
                             yield entry
 
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
         spider = self.spider_class("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/english/article_1/',
-                          'http://www.example.com/english/article_2/'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            [
+                "http://www.example.com/english/article_1/",
+                "http://www.example.com/english/article_2/",
+            ],
+        )
 
         spider = FilteredSitemapSpider("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/deutsch/article_1/'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            ["http://www.example.com/deutsch/article_1/"],
+        )
 
     def test_sitemapindex_filter(self):
         sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -581,27 +657,35 @@ Sitemap: /sitemap-relative-url.xml
         class FilteredSitemapSpider(self.spider_class):
             def sitemap_filter(self, entries):
                 from datetime import datetime
+
                 for entry in entries:
-                    date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d')
+                    date_time = datetime.strptime(
+                        entry["lastmod"].split("T")[0], "%Y-%m-%d"
+                    )
                     if date_time.year > 2004:
                         yield entry
 
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
         spider = self.spider_class("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/sitemap1.xml',
-                          'http://www.example.com/sitemap2.xml'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            [
+                "http://www.example.com/sitemap1.xml",
+                "http://www.example.com/sitemap2.xml",
+            ],
+        )
 
         spider = FilteredSitemapSpider("example.com")
-        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
-                         ['http://www.example.com/sitemap2.xml'])
+        self.assertEqual(
+            [req.url for req in spider._parse_sitemap(r)],
+            ["http://www.example.com/sitemap2.xml"],
+        )
 
 
 class DeprecationTest(unittest.TestCase):
-
     def test_crawl_spider(self):
         assert issubclass(CrawlSpider, Spider)
-        assert isinstance(CrawlSpider(name='foo'), Spider)
+        assert isinstance(CrawlSpider(name="foo"), Spider)
 
 
 class NoParseMethodSpiderTest(unittest.TestCase):
@@ -609,10 +693,10 @@ class NoParseMethodSpiderTest(unittest.TestCase):
     spider_class = Spider
 
     def test_undefined_parse_method(self):
-        spider = self.spider_class('example.com')
-        text = b'Random text'
+        spider = self.spider_class("example.com")
+        text = b"Random text"
         resp = TextResponse(url="http://www.example.com/random_url", body=text)
 
-        exc_msg = 'Spider.parse callback is not defined'
+        exc_msg = "Spider.parse callback is not defined"
         with self.assertRaisesRegex(NotImplementedError, exc_msg):
             spider.parse(resp)
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 0b6b51a5b..3745355a0 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -28,19 +28,18 @@ def _copytree(source: Path, target: Path):
 
 
 class SpiderLoaderTest(unittest.TestCase):
-
     def setUp(self):
-        orig_spiders_dir = module_dir / 'test_spiders'
+        orig_spiders_dir = module_dir / "test_spiders"
         self.tmpdir = Path(tempfile.mkdtemp())
-        self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
+        self.spiders_dir = self.tmpdir / "test_spiders_xxx"
         _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(str(self.tmpdir))
-        settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+        settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
         self.spider_loader = SpiderLoader.from_settings(settings)
 
     def tearDown(self):
         del self.spider_loader
-        del sys.modules['test_spiders_xxx']
+        del sys.modules["test_spiders_xxx"]
         sys.path.remove(str(self.tmpdir))
 
     def test_interface(self):
@@ -48,78 +47,86 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def test_list(self):
         self.assertEqual(
-            set(self.spider_loader.list()),
-            {'spider1', 'spider2', 'spider3', 'spider4'})
+            set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}
+        )
 
     def test_load(self):
         spider1 = self.spider_loader.load("spider1")
-        self.assertEqual(spider1.__name__, 'Spider1')
+        self.assertEqual(spider1.__name__, "Spider1")
 
     def test_find_by_request(self):
         self.assertEqual(
-            self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
-            ['spider1'])
+            self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),
+            ["spider1"],
+        )
         self.assertEqual(
-            self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
-            ['spider2'])
+            self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),
+            ["spider2"],
+        )
         self.assertEqual(
-            set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
-            {'spider1', 'spider2'})
+            set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),
+            {"spider1", "spider2"},
+        )
         self.assertEqual(
-            self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
-            [])
+            self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []
+        )
         self.assertEqual(
-            self.spider_loader.find_by_request(Request('http://spider3.com')),
-            [])
+            self.spider_loader.find_by_request(Request("http://spider3.com")), []
+        )
         self.assertEqual(
-            self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
-            ['spider3'])
+            self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),
+            ["spider3"],
+        )
 
     def test_load_spider_module(self):
-        module = 'tests.test_spiderloader.test_spiders.spider1'
-        settings = Settings({'SPIDER_MODULES': [module]})
+        module = "tests.test_spiderloader.test_spiders.spider1"
+        settings = Settings({"SPIDER_MODULES": [module]})
         self.spider_loader = SpiderLoader.from_settings(settings)
         assert len(self.spider_loader._spiders) == 1
 
     def test_load_spider_module_multiple(self):
-        prefix = 'tests.test_spiderloader.test_spiders.'
-        module = ','.join(prefix + s for s in ('spider1', 'spider2'))
-        settings = Settings({'SPIDER_MODULES': module})
+        prefix = "tests.test_spiderloader.test_spiders."
+        module = ",".join(prefix + s for s in ("spider1", "spider2"))
+        settings = Settings({"SPIDER_MODULES": module})
         self.spider_loader = SpiderLoader.from_settings(settings)
         assert len(self.spider_loader._spiders) == 2
 
     def test_load_base_spider(self):
-        module = 'tests.test_spiderloader.test_spiders.spider0'
-        settings = Settings({'SPIDER_MODULES': [module]})
+        module = "tests.test_spiderloader.test_spiders.spider0"
+        settings = Settings({"SPIDER_MODULES": [module]})
         self.spider_loader = SpiderLoader.from_settings(settings)
         assert len(self.spider_loader._spiders) == 0
 
     def test_crawler_runner_loading(self):
-        module = 'tests.test_spiderloader.test_spiders.spider1'
-        runner = CrawlerRunner({
-            'SPIDER_MODULES': [module],
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
-        })
+        module = "tests.test_spiderloader.test_spiders.spider1"
+        runner = CrawlerRunner(
+            {
+                "SPIDER_MODULES": [module],
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+            }
+        )
 
-        self.assertRaisesRegex(KeyError, 'Spider not found',
-                               runner.create_crawler, 'spider2')
+        self.assertRaisesRegex(
+            KeyError, "Spider not found", runner.create_crawler, "spider2"
+        )
 
-        crawler = runner.create_crawler('spider1')
+        crawler = runner.create_crawler("spider1")
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
-        self.assertEqual(crawler.spidercls.name, 'spider1')
+        self.assertEqual(crawler.spidercls.name, "spider1")
 
     def test_bad_spider_modules_exception(self):
 
-        module = 'tests.test_spiderloader.test_spiders.doesnotexist'
-        settings = Settings({'SPIDER_MODULES': [module]})
+        module = "tests.test_spiderloader.test_spiders.doesnotexist"
+        settings = Settings({"SPIDER_MODULES": [module]})
         self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
 
     def test_bad_spider_modules_warning(self):
 
         with warnings.catch_warnings(record=True) as w:
-            module = 'tests.test_spiderloader.test_spiders.doesnotexist'
-            settings = Settings({'SPIDER_MODULES': [module],
-                                 'SPIDER_LOADER_WARN_ONLY': True})
+            module = "tests.test_spiderloader.test_spiders.doesnotexist"
+            settings = Settings(
+                {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
+            )
             spider_loader = SpiderLoader.from_settings(settings)
             if str(w[0].message).startswith("_SixMetaPathImporter"):
                 # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
@@ -133,24 +140,25 @@ class SpiderLoaderTest(unittest.TestCase):
 
 
 class DuplicateSpiderNameLoaderTest(unittest.TestCase):
-
     def setUp(self):
-        orig_spiders_dir = module_dir / 'test_spiders'
+        orig_spiders_dir = module_dir / "test_spiders"
         self.tmpdir = Path(self.mktemp())
         self.tmpdir.mkdir()
-        self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
+        self.spiders_dir = self.tmpdir / "test_spiders_xxx"
         _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(str(self.tmpdir))
-        self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+        self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
 
     def tearDown(self):
-        del sys.modules['test_spiders_xxx']
+        del sys.modules["test_spiders_xxx"]
         sys.path.remove(str(self.tmpdir))
 
     def test_dupename_warning(self):
         # copy 1 spider module so as to have duplicate spider name
-        shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py',
-                        self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py')
+        shutil.copyfile(
+            self.tmpdir / "test_spiders_xxx" / "spider3.py",
+            self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",
+        )
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
@@ -166,15 +174,19 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
+            self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
 
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
         # This should issue 2 warning, 1 for each duplicate spider name
-        shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py',
-                        self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py')
-        shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py',
-                        self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py')
+        shutil.copyfile(
+            self.tmpdir / "test_spiders_xxx" / "spider1.py",
+            self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",
+        )
+        shutil.copyfile(
+            self.tmpdir / "test_spiders_xxx" / "spider2.py",
+            self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",
+        )
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
@@ -192,4 +204,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
+            self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py
index dbd1fb123..6c8ea4fe4 100644
--- a/tests/test_spiderloader/test_spiders/nested/spider4.py
+++ b/tests/test_spiderloader/test_spiders/nested/spider4.py
@@ -3,8 +3,8 @@ from scrapy.spiders import Spider
 
 class Spider4(Spider):
     name = "spider4"
-    allowed_domains = ['spider4.com']
+    allowed_domains = ["spider4.com"]
 
     @classmethod
     def handles_request(cls, request):
-        return request.url == 'http://spider4.com/onlythis'
+        return request.url == "http://spider4.com/onlythis"
diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py
index 84998ba35..d99878276 100644
--- a/tests/test_spiderloader/test_spiders/spider3.py
+++ b/tests/test_spiderloader/test_spiders/spider3.py
@@ -3,8 +3,8 @@ from scrapy.spiders import Spider
 
 class Spider3(Spider):
     name = "spider3"
-    allowed_domains = ['spider3.com']
+    allowed_domains = ["spider3.com"]
 
     @classmethod
     def handles_request(cls, request):
-        return request.url == 'http://spider3.com/onlythis'
+        return request.url == "http://spider3.com/onlythis"
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index edde6f682..760ee43df 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -17,12 +17,11 @@ from scrapy.core.spidermw import SpiderMiddlewareManager
 
 
 class SpiderMiddlewareTestCase(TestCase):
-
     def setUp(self):
-        self.request = Request('http://example.com/index.html')
+        self.request = Request("http://example.com/index.html")
         self.response = Response(self.request.url, request=self.request)
-        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}})
-        self.spider = self.crawler._create_spider('foo')
+        self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
+        self.spider = self.crawler._create_spider("foo")
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
 
     def _scrape_response(self):
@@ -30,7 +29,9 @@ class SpiderMiddlewareTestCase(TestCase):
         Raise exception in case of failure.
         """
         scrape_func = mock.MagicMock()
-        dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider)
+        dfd = self.mwman.scrape_response(
+            scrape_func, self.response, self.request, self.spider
+        )
         # catch deferred result and return the value
         results = []
         dfd.addBoth(results.append)
@@ -43,7 +44,6 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
     """Invalid return value for process_spider_input method"""
 
     def test_invalid_process_spider_input(self):
-
         class InvalidProcessSpiderInputMiddleware:
             def process_spider_input(self, response, spider):
                 return 1
@@ -58,7 +58,6 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
     """Invalid return value for process_spider_output method"""
 
     def test_invalid_process_spider_output(self):
-
         class InvalidProcessSpiderOutputMiddleware:
             def process_spider_output(self, response, result, spider):
                 return 1
@@ -73,7 +72,6 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
     """Invalid return value for process_spider_exception method"""
 
     def test_invalid_process_spider_exception(self):
-
         class InvalidProcessSpiderOutputExceptionMiddleware:
             def process_spider_exception(self, response, exception, spider):
                 return 1
@@ -93,7 +91,6 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
     """Re raise the exception by returning None"""
 
     def test_process_spider_exception_return_none(self):
-
         class ProcessSpiderExceptionReturnNoneMiddleware:
             def process_spider_exception(self, response, exception, spider):
                 return None
@@ -110,7 +107,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
 
 
 class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
-    """ Helpers for testing sync, async and mixed middlewares.
+    """Helpers for testing sync, async and mixed middlewares.
 
     Should work for process_spider_output and, when it's supported, process_start_requests.
     """
@@ -124,23 +121,31 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
         return {i: c for c, i in enumerate(mw_classes, start=start_index)}
 
     def _scrape_func(self, *args, **kwargs):
-        yield {'foo': 1}
-        yield {'foo': 2}
-        yield {'foo': 3}
+        yield {"foo": 1}
+        yield {"foo": 2}
+        yield {"foo": 3}
 
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
-        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
-        self.spider = self.crawler._create_spider('foo')
+        self.crawler = get_crawler(
+            Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
+        )
+        self.spider = self.crawler._create_spider("foo")
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
-        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        result = yield self.mwman.scrape_response(
+            self._scrape_func, self.response, self.request, self.spider
+        )
         return result
 
     @defer.inlineCallbacks
-    def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+    def _test_simple_base(
+        self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None
+    ):
         with LogCapture() as log:
-            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
+            result = yield self._get_middleware_result(
+                *mw_classes, start_index=start_index
+            )
         self.assertIsInstance(result, collections.abc.Iterable)
         result_list = list(result)
         self.assertEqual(len(result_list), self.RESULT_COUNT)
@@ -148,9 +153,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
         self.assertEqual("downgraded to a non-async" in str(log), downgrade)
 
     @defer.inlineCallbacks
-    def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+    def _test_asyncgen_base(
+        self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None
+    ):
         with LogCapture() as log:
-            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
+            result = yield self._get_middleware_result(
+                *mw_classes, start_index=start_index
+            )
         self.assertIsInstance(result, collections.abc.AsyncIterator)
         result_list = yield deferred_from_coro(collect_asyncgen(result))
         self.assertEqual(len(result_list), self.RESULT_COUNT)
@@ -182,24 +191,25 @@ class ProcessSpiderOutputUniversalMiddleware:
 
 class ProcessSpiderExceptionSimpleIterableMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        yield {'foo': 1}
-        yield {'foo': 2}
-        yield {'foo': 3}
+        yield {"foo": 1}
+        yield {"foo": 2}
+        yield {"foo": 3}
 
 
 class ProcessSpiderExceptionAsyncIterableMiddleware:
     async def process_spider_exception(self, response, exception, spider):
-        yield {'foo': 1}
+        yield {"foo": 1}
         d = defer.Deferred()
         from twisted.internet import reactor
+
         reactor.callLater(0, d.callback, None)
         await maybe_deferred_to_future(d)
-        yield {'foo': 2}
-        yield {'foo': 3}
+        yield {"foo": 2}
+        yield {"foo": 3}
 
 
 class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
-    """ process_spider_output tests for simple callbacks"""
+    """process_spider_output tests for simple callbacks"""
 
     ITEM_TYPE = dict
     MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
@@ -207,82 +217,70 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
     def test_simple(self):
-        """ Simple mw """
+        """Simple mw"""
         return self._test_simple_base(self.MW_SIMPLE)
 
     def test_asyncgen(self):
-        """ Asyncgen mw; upgrade """
+        """Asyncgen mw; upgrade"""
         return self._test_asyncgen_base(self.MW_ASYNCGEN)
 
     def test_simple_asyncgen(self):
-        """ Simple mw -> asyncgen mw; upgrade """
-        return self._test_asyncgen_base(self.MW_ASYNCGEN,
-                                        self.MW_SIMPLE)
+        """Simple mw -> asyncgen mw; upgrade"""
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
 
     def test_asyncgen_simple(self):
-        """ Asyncgen mw -> simple mw; upgrade then downgrade """
-        return self._test_simple_base(self.MW_SIMPLE,
-                                      self.MW_ASYNCGEN,
-                                      downgrade=True)
+        """Asyncgen mw -> simple mw; upgrade then downgrade"""
+        return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
 
     def test_universal(self):
-        """ Universal mw """
+        """Universal mw"""
         return self._test_simple_base(self.MW_UNIVERSAL)
 
     def test_universal_simple(self):
-        """ Universal mw -> simple mw """
-        return self._test_simple_base(self.MW_SIMPLE,
-                                      self.MW_UNIVERSAL)
+        """Universal mw -> simple mw"""
+        return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
 
     def test_simple_universal(self):
-        """ Simple mw -> universal mw """
-        return self._test_simple_base(self.MW_UNIVERSAL,
-                                      self.MW_SIMPLE)
+        """Simple mw -> universal mw"""
+        return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
 
     def test_universal_asyncgen(self):
-        """ Universal mw -> asyncgen mw; upgrade """
-        return self._test_asyncgen_base(self.MW_ASYNCGEN,
-                                        self.MW_UNIVERSAL)
+        """Universal mw -> asyncgen mw; upgrade"""
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
 
     def test_asyncgen_universal(self):
-        """ Asyncgen mw -> universal mw; upgrade """
-        return self._test_asyncgen_base(self.MW_UNIVERSAL,
-                                        self.MW_ASYNCGEN)
+        """Asyncgen mw -> universal mw; upgrade"""
+        return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
 
 
 class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
-    """ process_spider_output tests for async generator callbacks """
+    """process_spider_output tests for async generator callbacks"""
 
     async def _scrape_func(self, *args, **kwargs):
         for item in super()._scrape_func():
             yield item
 
     def test_simple(self):
-        """ Simple mw; downgrade """
-        return self._test_simple_base(self.MW_SIMPLE,
-                                      downgrade=True)
+        """Simple mw; downgrade"""
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
 
     def test_simple_asyncgen(self):
-        """ Simple mw -> asyncgen mw; downgrade then upgrade """
-        return self._test_asyncgen_base(self.MW_ASYNCGEN,
-                                        self.MW_SIMPLE,
-                                        downgrade=True)
+        """Simple mw -> asyncgen mw; downgrade then upgrade"""
+        return self._test_asyncgen_base(
+            self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True
+        )
 
     def test_universal(self):
-        """ Universal mw """
+        """Universal mw"""
         return self._test_asyncgen_base(self.MW_UNIVERSAL)
 
     def test_universal_simple(self):
-        """ Universal mw -> simple mw; downgrade """
-        return self._test_simple_base(self.MW_SIMPLE,
-                                      self.MW_UNIVERSAL,
-                                      downgrade=True)
+        """Universal mw -> simple mw; downgrade"""
+        return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
 
     def test_simple_universal(self):
-        """ Simple mw -> universal mw; downgrade """
-        return self._test_simple_base(self.MW_UNIVERSAL,
-                                      self.MW_SIMPLE,
-                                      downgrade=True)
+        """Simple mw -> universal mw; downgrade"""
+        return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
 
 
 class ProcessSpiderOutputNonIterableMiddleware:
@@ -299,7 +297,6 @@ class ProcessSpiderOutputCoroutineMiddleware:
 
 
 class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
-
     @defer.inlineCallbacks
     def test_non_iterable(self):
         with self.assertRaisesRegex(
@@ -331,27 +328,29 @@ class ProcessStartRequestsSimpleMiddleware:
 
 
 class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
-    """ process_start_requests tests for simple start_requests"""
+    """process_start_requests tests for simple start_requests"""
 
     ITEM_TYPE = Request
     MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
 
     def _start_requests(self):
         for i in range(3):
-            yield Request(f'https://example.com/{i}', dont_filter=True)
+            yield Request(f"https://example.com/{i}", dont_filter=True)
 
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
-        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
-        self.spider = self.crawler._create_spider('foo')
+        self.crawler = get_crawler(
+            Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
+        )
+        self.spider = self.crawler._create_spider("foo")
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
         start_requests = iter(self._start_requests())
         results = yield self.mwman.process_start_requests(start_requests, self.spider)
         return results
 
     def test_simple(self):
-        """ Simple mw """
+        """Simple mw"""
         return self._test_simple_base(self.MW_SIMPLE)
 
 
@@ -383,41 +382,57 @@ class UniversalMiddlewareManagerTest(TestCase):
     def test_simple_mw(self):
         mw = ProcessSpiderOutputSimpleMiddleware
         self.mwman._add_middleware(mw)
-        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+        self.assertEqual(
+            self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+        )
 
     def test_async_mw(self):
         mw = ProcessSpiderOutputAsyncGenMiddleware
         self.mwman._add_middleware(mw)
-        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+        self.assertEqual(
+            self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+        )
 
     def test_universal_mw(self):
         mw = ProcessSpiderOutputUniversalMiddleware
         self.mwman._add_middleware(mw)
-        self.assertEqual(self.mwman.methods['process_spider_output'][0],
-                         (mw.process_spider_output, mw.process_spider_output_async))
+        self.assertEqual(
+            self.mwman.methods["process_spider_output"][0],
+            (mw.process_spider_output, mw.process_spider_output_async),
+        )
 
     def test_universal_mw_no_sync(self):
         with LogCapture() as log:
             self.mwman._add_middleware(UniversalMiddlewareNoSync)
-        self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async"
-                      " without process_spider_output", str(log))
-        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+        self.assertIn(
+            "UniversalMiddlewareNoSync has process_spider_output_async"
+            " without process_spider_output",
+            str(log),
+        )
+        self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
 
     def test_universal_mw_both_sync(self):
         mw = UniversalMiddlewareBothSync
         with LogCapture() as log:
             self.mwman._add_middleware(mw)
-        self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async "
-                      "is not an async generator function", str(log))
-        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+        self.assertIn(
+            "UniversalMiddlewareBothSync.process_spider_output_async "
+            "is not an async generator function",
+            str(log),
+        )
+        self.assertEqual(
+            self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+        )
 
     def test_universal_mw_both_async(self):
         with LogCapture() as log:
             self.mwman._add_middleware(UniversalMiddlewareBothAsync)
-        self.assertIn("UniversalMiddlewareBothAsync.process_spider_output "
-                      "is an async generator function while process_spider_output_async exists",
-                      str(log))
-        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+        self.assertIn(
+            "UniversalMiddlewareBothAsync.process_spider_output "
+            "is an async generator function while process_spider_output_async exists",
+            str(log),
+        )
+        self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
 
 
 class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
@@ -429,10 +444,12 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
-        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting})
-        self.spider = self.crawler._create_spider('foo')
+        self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
+        self.spider = self.crawler._create_spider("foo")
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
-        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        result = yield self.mwman.scrape_response(
+            self._scrape_func, self.response, self.request, self.spider
+        )
         return result
 
     def test_just_builtin(self):
@@ -442,7 +459,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
         return self._test_simple_base(self.MW_SIMPLE, start_index=1000)
 
     def test_builtin_async(self):
-        """ Upgrade """
+        """Upgrade"""
         return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
 
     def test_builtin_universal(self):
@@ -452,7 +469,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
         return self._test_simple_base(self.MW_SIMPLE)
 
     def test_async_builtin(self):
-        """ Upgrade """
+        """Upgrade"""
         return self._test_asyncgen_base(self.MW_ASYNCGEN)
 
     def test_universal_builtin(self):
@@ -468,7 +485,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
         return self._test_asyncgen_base()
 
     def test_builtin_simple(self):
-        """ Downgrade """
+        """Downgrade"""
         return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
 
     def test_builtin_async(self):
@@ -478,7 +495,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
         return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
 
     def test_simple_builtin(self):
-        """ Downgrade """
+        """Downgrade"""
         return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
 
     def test_async_builtin(self):
@@ -501,33 +518,31 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
 
     @defer.inlineCallbacks
     def _test_asyncgen_nodowngrade(self, *mw_classes):
-        with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"):
+        with self.assertRaisesRegex(
+            _InvalidOutput, "Async iterable returned from .+ cannot be downgraded"
+        ):
             yield self._get_middleware_result(*mw_classes)
 
     def test_exc_simple(self):
-        """ Simple exc mw """
+        """Simple exc mw"""
         return self._test_simple_base(self.MW_EXC_SIMPLE)
 
     def test_exc_async(self):
-        """ Async exc mw """
+        """Async exc mw"""
         return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
 
     def test_exc_simple_simple(self):
-        """ Simple exc mw -> simple output mw """
-        return self._test_simple_base(self.MW_SIMPLE,
-                                      self.MW_EXC_SIMPLE)
+        """Simple exc mw -> simple output mw"""
+        return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
 
     def test_exc_async_async(self):
-        """ Async exc mw -> async output mw """
-        return self._test_asyncgen_base(self.MW_ASYNCGEN,
-                                        self.MW_EXC_ASYNCGEN)
+        """Async exc mw -> async output mw"""
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN)
 
     def test_exc_simple_async(self):
-        """ Simple exc mw -> async output mw; upgrade """
-        return self._test_asyncgen_base(self.MW_ASYNCGEN,
-                                        self.MW_EXC_SIMPLE)
+        """Simple exc mw -> async output mw; upgrade"""
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
 
     def test_exc_async_simple(self):
-        """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """
-        return self._test_asyncgen_nodowngrade(self.MW_SIMPLE,
-                                               self.MW_EXC_ASYNCGEN)
+        """Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
+        return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 71cca2472..af17c13a0 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -8,10 +8,9 @@ from scrapy.utils.test import get_crawler
 
 
 class TestDepthMiddleware(TestCase):
-
     def setUp(self):
         crawler = get_crawler(Spider)
-        self.spider = crawler._create_spider('scrapytest.org')
+        self.spider = crawler._create_spider("scrapytest.org")
 
         self.stats = StatsCollector(crawler)
         self.stats.open_spider(self.spider)
@@ -19,24 +18,24 @@ class TestDepthMiddleware(TestCase):
         self.mw = DepthMiddleware(1, self.stats, True)
 
     def test_process_spider_output(self):
-        req = Request('http://scrapytest.org')
-        resp = Response('http://scrapytest.org')
+        req = Request("http://scrapytest.org")
+        resp = Response("http://scrapytest.org")
         resp.request = req
-        result = [Request('http://scrapytest.org')]
+        result = [Request("http://scrapytest.org")]
 
         out = list(self.mw.process_spider_output(resp, result, self.spider))
         self.assertEqual(out, result)
 
-        rdc = self.stats.get_value('request_depth_count/1', spider=self.spider)
+        rdc = self.stats.get_value("request_depth_count/1", spider=self.spider)
         self.assertEqual(rdc, 1)
 
-        req.meta['depth'] = 1
+        req.meta["depth"] = 1
 
         out2 = list(self.mw.process_spider_output(resp, result, self.spider))
         self.assertEqual(out2, [])
 
-        rdm = self.stats.get_value('request_depth_max', spider=self.spider)
+        rdm = self.stats.get_value("request_depth_max", spider=self.spider)
         self.assertEqual(rdm, 1)
 
     def tearDown(self):
-        self.stats.close_spider(self.spider, '')
+        self.stats.close_spider(self.spider, "")
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index ee11ee492..faa8e9091 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -15,7 +15,7 @@ from tests.spiders import MockServerSpider
 
 
 class _HttpErrorSpider(MockServerSpider):
-    name = 'httperror'
+    name = "httperror"
     bypass_status_codes = set()
 
     def __init__(self, *args, **kwargs):
@@ -59,28 +59,35 @@ def _responses(request, status_codes):
 
 
 class TestHttpErrorMiddleware(TestCase):
-
     def setUp(self):
         crawler = get_crawler(Spider)
-        self.spider = Spider.from_crawler(crawler, name='foo')
+        self.spider = Spider.from_crawler(crawler, name="foo")
         self.mw = HttpErrorMiddleware(Settings({}))
-        self.req = Request('http://scrapytest.org')
+        self.req = Request("http://scrapytest.org")
         self.res200, self.res404 = _responses(self.req, [200, 404])
 
     def test_process_spider_input(self):
         self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
+        self.assertRaises(
+            HttpError, self.mw.process_spider_input, self.res404, self.spider
+        )
 
     def test_process_spider_exception(self):
         self.assertEqual(
             [],
-            self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
-        self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
+            self.mw.process_spider_exception(
+                self.res404, HttpError(self.res404), self.spider
+            ),
+        )
+        self.assertIsNone(
+            self.mw.process_spider_exception(self.res404, Exception(), self.spider)
+        )
 
     def test_handle_httpstatus_list(self):
         res = self.res404.copy()
-        res.request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        res.request = Request(
+            "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
+        )
         self.assertIsNone(self.mw.process_spider_input(res, self.spider))
 
         self.spider.handle_httpstatus_list = [404]
@@ -91,18 +98,22 @@ class TestHttpErrorMiddlewareSettings(TestCase):
     """Similar test, but with settings"""
 
     def setUp(self):
-        self.spider = Spider('foo')
-        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)}))
-        self.req = Request('http://scrapytest.org')
+        self.spider = Spider("foo")
+        self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
+        self.req = Request("http://scrapytest.org")
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
         self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
+        self.assertRaises(
+            HttpError, self.mw.process_spider_input, self.res404, self.spider
+        )
         self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
+        request = Request(
+            "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
+        )
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
@@ -114,15 +125,16 @@ class TestHttpErrorMiddlewareSettings(TestCase):
     def test_spider_override_settings(self):
         self.spider.handle_httpstatus_list = [404]
         self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
-        self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
+        self.assertRaises(
+            HttpError, self.mw.process_spider_input, self.res402, self.spider
+        )
 
 
 class TestHttpErrorMiddlewareHandleAll(TestCase):
-
     def setUp(self):
-        self.spider = Spider('foo')
-        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
-        self.req = Request('http://scrapytest.org')
+        self.spider = Spider("foo")
+        self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
+        self.req = Request("http://scrapytest.org")
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
@@ -130,7 +142,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
+        request = Request(
+            "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
+        )
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
@@ -142,8 +156,12 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
     def test_httperror_allow_all_false(self):
         crawler = get_crawler(_HttpErrorSpider)
         mw = HttpErrorMiddleware.from_crawler(crawler)
-        request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
-        request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
+        request_httpstatus_false = Request(
+            "http://scrapytest.org", meta={"handle_httpstatus_all": False}
+        )
+        request_httpstatus_true = Request(
+            "http://scrapytest.org", meta={"handle_httpstatus_all": True}
+        )
         res404 = self.res404.copy()
         res404.request = request_httpstatus_false
         res402 = self.res402.copy()
@@ -166,28 +184,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
         crawler = get_crawler(_HttpErrorSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         assert not crawler.spider.skipped, crawler.spider.skipped
-        self.assertEqual(crawler.spider.parsed, {'200'})
-        self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+        self.assertEqual(crawler.spider.parsed, {"200"})
+        self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
 
         get_value = crawler.stats.get_value
-        self.assertEqual(get_value('httperror/response_ignored_count'), 3)
-        self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)
-        self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)
-        self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)
+        self.assertEqual(get_value("httperror/response_ignored_count"), 3)
+        self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1)
+        self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1)
+        self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1)
 
     @defer.inlineCallbacks
     def test_logging(self):
         crawler = get_crawler(_HttpErrorSpider)
         with LogCapture() as log:
             yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
-        self.assertEqual(crawler.spider.parsed, {'200', '402'})
-        self.assertEqual(crawler.spider.skipped, {'402'})
-        self.assertEqual(crawler.spider.failed, {'404', '500'})
+        self.assertEqual(crawler.spider.parsed, {"200", "402"})
+        self.assertEqual(crawler.spider.skipped, {"402"})
+        self.assertEqual(crawler.spider.failed, {"404", "500"})
 
-        self.assertIn('Ignoring response <404', str(log))
-        self.assertIn('Ignoring response <500', str(log))
-        self.assertNotIn('Ignoring response <200', str(log))
-        self.assertNotIn('Ignoring response <402', str(log))
+        self.assertIn("Ignoring response <404", str(log))
+        self.assertIn("Ignoring response <500", str(log))
+        self.assertNotIn("Ignoring response <200", str(log))
+        self.assertNotIn("Ignoring response <402", str(log))
 
     @defer.inlineCallbacks
     def test_logging_level(self):
@@ -195,22 +213,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
         crawler = get_crawler(_HttpErrorSpider)
         with LogCapture(level=logging.INFO) as log:
             yield crawler.crawl(mockserver=self.mockserver)
-        self.assertEqual(crawler.spider.parsed, {'200'})
-        self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+        self.assertEqual(crawler.spider.parsed, {"200"})
+        self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
 
-        self.assertIn('Ignoring response <402', str(log))
-        self.assertIn('Ignoring response <404', str(log))
-        self.assertIn('Ignoring response <500', str(log))
-        self.assertNotIn('Ignoring response <200', str(log))
+        self.assertIn("Ignoring response <402", str(log))
+        self.assertIn("Ignoring response <404", str(log))
+        self.assertIn("Ignoring response <500", str(log))
+        self.assertNotIn("Ignoring response <200", str(log))
 
         # with level WARNING, we shouldn't capture anything from HttpError
         crawler = get_crawler(_HttpErrorSpider)
         with LogCapture(level=logging.WARNING) as log:
             yield crawler.crawl(mockserver=self.mockserver)
-        self.assertEqual(crawler.spider.parsed, {'200'})
-        self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+        self.assertEqual(crawler.spider.parsed, {"200"})
+        self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
 
-        self.assertNotIn('Ignoring response <402', str(log))
-        self.assertNotIn('Ignoring response <404', str(log))
-        self.assertNotIn('Ignoring response <500', str(log))
-        self.assertNotIn('Ignoring response <200', str(log))
+        self.assertNotIn("Ignoring response <402", str(log))
+        self.assertNotIn("Ignoring response <404", str(log))
+        self.assertNotIn("Ignoring response <500", str(log))
+        self.assertNotIn("Ignoring response <200", str(log))
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 0f4b98a07..380bafe04 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -9,7 +9,6 @@ from scrapy.utils.test import get_crawler
 
 
 class TestOffsiteMiddleware(TestCase):
-
     def setUp(self):
         crawler = get_crawler(Spider)
         self.spider = crawler._create_spider(**self._get_spiderargs())
@@ -17,28 +16,31 @@ class TestOffsiteMiddleware(TestCase):
         self.mw.spider_opened(self.spider)
 
     def _get_spiderargs(self):
-        return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org'])
+        return dict(
+            name="foo",
+            allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"],
+        )
 
     def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
+        res = Response("http://scrapytest.org")
 
         onsite_reqs = [
-            Request('http://scrapytest.org/1'),
-            Request('http://scrapy.org/1'),
-            Request('http://sub.scrapy.org/1'),
-            Request('http://offsite.tld/letmepass', dont_filter=True),
-            Request('http://scrapy.test.org/'),
-            Request('http://scrapy.test.org:8000/'),
+            Request("http://scrapytest.org/1"),
+            Request("http://scrapy.org/1"),
+            Request("http://sub.scrapy.org/1"),
+            Request("http://offsite.tld/letmepass", dont_filter=True),
+            Request("http://scrapy.test.org/"),
+            Request("http://scrapy.test.org:8000/"),
         ]
         offsite_reqs = [
-            Request('http://scrapy2.org'),
-            Request('http://offsite.tld/'),
-            Request('http://offsite.tld/scrapytest.org'),
-            Request('http://offsite.tld/rogue.scrapytest.org'),
-            Request('http://rogue.scrapytest.org.haha.com'),
-            Request('http://roguescrapytest.org'),
-            Request('http://test.org/'),
-            Request('http://notscrapy.test.org/'),
+            Request("http://scrapy2.org"),
+            Request("http://offsite.tld/"),
+            Request("http://offsite.tld/scrapytest.org"),
+            Request("http://offsite.tld/rogue.scrapytest.org"),
+            Request("http://rogue.scrapytest.org.haha.com"),
+            Request("http://roguescrapytest.org"),
+            Request("http://test.org/"),
+            Request("http://notscrapy.test.org/"),
         ]
         reqs = onsite_reqs + offsite_reqs
 
@@ -47,40 +49,40 @@ class TestOffsiteMiddleware(TestCase):
 
 
 class TestOffsiteMiddleware2(TestOffsiteMiddleware):
-
     def _get_spiderargs(self):
-        return dict(name='foo', allowed_domains=None)
+        return dict(name="foo", allowed_domains=None)
 
     def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
-        reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')]
+        res = Response("http://scrapytest.org")
+        reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
         self.assertEqual(out, reqs)
 
 
 class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
-
     def _get_spiderargs(self):
-        return dict(name='foo')
+        return dict(name="foo")
 
 
 class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
-
     def _get_spiderargs(self):
-        bad_hostname = urlparse('http:////scrapytest.org').hostname
-        return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
+        bad_hostname = urlparse("http:////scrapytest.org").hostname
+        return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname])
 
     def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
-        reqs = [Request('http://scrapytest.org/1')]
+        res = Response("http://scrapytest.org")
+        reqs = [Request("http://scrapytest.org/1")]
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
         self.assertEqual(out, reqs)
 
 
 class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
-
     def test_get_host_regex(self):
-        self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']
+        self.spider.allowed_domains = [
+            "http://scrapytest.org",
+            "scrapy.org",
+            "scrapy.test.org",
+        ]
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter("always")
             self.mw.get_host_regex(self.spider)
@@ -88,9 +90,12 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
 
 
 class TestOffsiteMiddleware6(TestOffsiteMiddleware4):
-
     def test_get_host_regex(self):
-        self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org']
+        self.spider.allowed_domains = [
+            "scrapytest.org:8000",
+            "scrapy.org",
+            "scrapy.test.org",
+        ]
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter("always")
             self.mw.get_host_regex(self.spider)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 8dd1def17..fad5dcaac 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -9,7 +9,9 @@ from tests.mockserver import MockServer
 
 class LogExceptionMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+        spider.logger.info(
+            "Middleware: %s exception caught", exception.__class__.__name__
+        )
         return None
 
 
@@ -17,34 +19,36 @@ class LogExceptionMiddleware:
 # (0) recover from an exception on a spider callback
 class RecoveryMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+        spider.logger.info(
+            "Middleware: %s exception caught", exception.__class__.__name__
+        )
         return [
-            {'from': 'process_spider_exception'},
-            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+            {"from": "process_spider_exception"},
+            Request(response.url, meta={"dont_fail": True}, dont_filter=True),
         ]
 
 
 class RecoverySpider(Spider):
-    name = 'RecoverySpider'
+    name = "RecoverySpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES_BASE': {},
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES_BASE": {},
+        "SPIDER_MIDDLEWARES": {
             RecoveryMiddleware: 10,
         },
     }
 
     def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
+        yield Request(self.mockserver.url("/status?n=200"))
 
     def parse(self, response):
-        yield {'test': 1}
-        self.logger.info('DONT_FAIL: %s', response.meta.get('dont_fail'))
-        if not response.meta.get('dont_fail'):
+        yield {"test": 1}
+        self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
+        if not response.meta.get("dont_fail"):
             raise TabError()
 
 
 class RecoveryAsyncGenSpider(RecoverySpider):
-    name = 'RecoveryAsyncGenSpider'
+    name = "RecoveryAsyncGenSpider"
 
     async def parse(self, response):
         for r in super().parse(response):
@@ -55,14 +59,14 @@ class RecoveryAsyncGenSpider(RecoverySpider):
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:
     def process_spider_input(self, response, spider):
-        spider.logger.info('Middleware: will raise IndexError')
+        spider.logger.info("Middleware: will raise IndexError")
         raise IndexError()
 
 
 class ProcessSpiderInputSpiderWithoutErrback(Spider):
-    name = 'ProcessSpiderInputSpiderWithoutErrback'
+    name = "ProcessSpiderInputSpiderWithoutErrback"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             # spider
             FailProcessSpiderInputMiddleware: 8,
             LogExceptionMiddleware: 6,
@@ -71,55 +75,57 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
     }
 
     def start_requests(self):
-        yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse)
+        yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)
 
     def parse(self, response):
-        return {'from': 'callback'}
+        return {"from": "callback"}
 
 
 class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
-    name = 'ProcessSpiderInputSpiderWithErrback'
+    name = "ProcessSpiderInputSpiderWithErrback"
 
     def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'), self.parse, errback=self.errback)
+        yield Request(
+            self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
+        )
 
     def errback(self, failure):
-        self.logger.info('Got a Failure on the Request errback')
-        return {'from': 'errback'}
+        self.logger.info("Got a Failure on the Request errback")
+        return {"from": "errback"}
 
 
 # ================================================================================
 # (2) exceptions from a spider callback (generator)
 class GeneratorCallbackSpider(Spider):
-    name = 'GeneratorCallbackSpider'
+    name = "GeneratorCallbackSpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             LogExceptionMiddleware: 10,
         },
     }
 
     def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
+        yield Request(self.mockserver.url("/status?n=200"))
 
     def parse(self, response):
-        yield {'test': 1}
-        yield {'test': 2}
+        yield {"test": 1}
+        yield {"test": 2}
         raise ImportError()
 
 
 class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
     async def parse(self, response):
-        yield {'test': 1}
-        yield {'test': 2}
+        yield {"test": 1}
+        yield {"test": 2}
         raise ImportError()
 
 
 # ================================================================================
 # (2.1) exceptions from a spider callback (generator, middleware right after callback)
 class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
-    name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
+    name = "GeneratorCallbackSpiderMiddlewareRightAfterSpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             LogExceptionMiddleware: 100000,
         },
     }
@@ -128,26 +134,26 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
 # ================================================================================
 # (3) exceptions from a spider callback (not a generator)
 class NotGeneratorCallbackSpider(Spider):
-    name = 'NotGeneratorCallbackSpider'
+    name = "NotGeneratorCallbackSpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             LogExceptionMiddleware: 10,
         },
     }
 
     def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
+        yield Request(self.mockserver.url("/status?n=200"))
 
     def parse(self, response):
-        return [{'test': 1}, {'test': 1 / 0}]
+        return [{"test": 1}, {"test": 1 / 0}]
 
 
 # ================================================================================
 # (3.1) exceptions from a spider callback (not a generator, middleware right after callback)
 class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider):
-    name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
+    name = "NotGeneratorCallbackSpiderMiddlewareRightAfterSpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             LogExceptionMiddleware: 100000,
         },
     }
@@ -158,26 +164,26 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
 class _GeneratorDoNothingMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
-            r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             yield r
 
     def process_spider_exception(self, response, exception, spider):
-        method = f'{self.__class__.__name__}.process_spider_exception'
-        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
+        method = f"{self.__class__.__name__}.process_spider_exception"
+        spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
         return None
 
 
 class GeneratorFailMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
-            r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             yield r
             raise LookupError()
 
     def process_spider_exception(self, response, exception, spider):
-        method = f'{self.__class__.__name__}.process_spider_exception'
-        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
-        yield {'processed': [method]}
+        method = f"{self.__class__.__name__}.process_spider_exception"
+        spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+        yield {"processed": [method]}
 
 
 class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
@@ -187,13 +193,13 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
 class GeneratorRecoverMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
-            r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             yield r
 
     def process_spider_exception(self, response, exception, spider):
-        method = f'{self.__class__.__name__}.process_spider_exception'
-        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
-        yield {'processed': [method]}
+        method = f"{self.__class__.__name__}.process_spider_exception"
+        spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+        yield {"processed": [method]}
 
 
 class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
@@ -201,9 +207,9 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
 
 
 class GeneratorOutputChainSpider(Spider):
-    name = 'GeneratorOutputChainSpider'
+    name = "GeneratorOutputChainSpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             GeneratorFailMiddleware: 10,
             GeneratorDoNothingAfterFailureMiddleware: 8,
             GeneratorRecoverMiddleware: 5,
@@ -212,27 +218,28 @@ class GeneratorOutputChainSpider(Spider):
     }
 
     def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
+        yield Request(self.mockserver.url("/status?n=200"))
 
     def parse(self, response):
-        yield {'processed': ['parse-first-item']}
-        yield {'processed': ['parse-second-item']}
+        yield {"processed": ["parse-first-item"]}
+        yield {"processed": ["parse-second-item"]}
 
 
 # ================================================================================
 # (5) exceptions from a middleware process_spider_output method (not generator)
 
+
 class _NotGeneratorDoNothingMiddleware:
     def process_spider_output(self, response, result, spider):
         out = []
         for r in result:
-            r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             out.append(r)
         return out
 
     def process_spider_exception(self, response, exception, spider):
-        method = f'{self.__class__.__name__}.process_spider_exception'
-        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
+        method = f"{self.__class__.__name__}.process_spider_exception"
+        spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
         return None
 
 
@@ -240,15 +247,15 @@ class NotGeneratorFailMiddleware:
     def process_spider_output(self, response, result, spider):
         out = []
         for r in result:
-            r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             out.append(r)
         raise ReferenceError()
         return out
 
     def process_spider_exception(self, response, exception, spider):
-        method = f'{self.__class__.__name__}.process_spider_exception'
-        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
-        return [{'processed': [method]}]
+        method = f"{self.__class__.__name__}.process_spider_exception"
+        spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+        return [{"processed": [method]}]
 
 
 class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware):
@@ -259,14 +266,14 @@ class NotGeneratorRecoverMiddleware:
     def process_spider_output(self, response, result, spider):
         out = []
         for r in result:
-            r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             out.append(r)
         return out
 
     def process_spider_exception(self, response, exception, spider):
-        method = f'{self.__class__.__name__}.process_spider_exception'
-        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
-        return [{'processed': [method]}]
+        method = f"{self.__class__.__name__}.process_spider_exception"
+        spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+        return [{"processed": [method]}]
 
 
 class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
@@ -274,9 +281,9 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew
 
 
 class NotGeneratorOutputChainSpider(Spider):
-    name = 'NotGeneratorOutputChainSpider'
+    name = "NotGeneratorOutputChainSpider"
     custom_settings = {
-        'SPIDER_MIDDLEWARES': {
+        "SPIDER_MIDDLEWARES": {
             NotGeneratorFailMiddleware: 10,
             NotGeneratorDoNothingAfterFailureMiddleware: 8,
             NotGeneratorRecoverMiddleware: 5,
@@ -285,10 +292,13 @@ class NotGeneratorOutputChainSpider(Spider):
     }
 
     def start_requests(self):
-        return [Request(self.mockserver.url('/status?n=200'))]
+        return [Request(self.mockserver.url("/status?n=200"))]
 
     def parse(self, response):
-        return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+        return [
+            {"processed": ["parse-first-item"]},
+            {"processed": ["parse-second-item"]},
+        ]
 
 
 # ================================================================================
@@ -402,7 +412,9 @@ class TestSpiderMiddleware(TestCase):
         (3.1) Special case of (3): Exceptions should be caught
         even if the middleware is placed right after the spider
         """
-        log31 = yield self.crawl_log(NotGeneratorCallbackSpiderMiddlewareRightAfterSpider)
+        log31 = yield self.crawl_log(
+            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
+        )
         self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))
         self.assertNotIn("item_scraped_count", str(log31))
 
@@ -418,28 +430,40 @@ class TestSpiderMiddleware(TestCase):
         """
         log4 = yield self.crawl_log(GeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 2", str(log4))
-        self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertIn(
+            "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught",
+            str(log4),
+        )
         self.assertIn(
             "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
-            str(log4))
+            str(log4),
+        )
         self.assertNotIn(
             "GeneratorFailMiddleware.process_spider_exception: LookupError caught",
-            str(log4))
+            str(log4),
+        )
         self.assertNotIn(
             "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
-            str(log4))
-        item_from_callback = {'processed': [
-            'parse-first-item',
-            'GeneratorFailMiddleware.process_spider_output',
-            'GeneratorDoNothingAfterFailureMiddleware.process_spider_output',
-            'GeneratorRecoverMiddleware.process_spider_output',
-            'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
-        item_recovered = {'processed': [
-            'GeneratorRecoverMiddleware.process_spider_exception',
-            'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+            str(log4),
+        )
+        item_from_callback = {
+            "processed": [
+                "parse-first-item",
+                "GeneratorFailMiddleware.process_spider_output",
+                "GeneratorDoNothingAfterFailureMiddleware.process_spider_output",
+                "GeneratorRecoverMiddleware.process_spider_output",
+                "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
+            ]
+        }
+        item_recovered = {
+            "processed": [
+                "GeneratorRecoverMiddleware.process_spider_exception",
+                "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
+            ]
+        }
         self.assertIn(str(item_from_callback), str(log4))
         self.assertIn(str(item_recovered), str(log4))
-        self.assertNotIn('parse-second-item', str(log4))
+        self.assertNotIn("parse-second-item", str(log4))
 
     @defer.inlineCallbacks
     def test_not_a_generator_output_chain(self):
@@ -453,17 +477,28 @@ class TestSpiderMiddleware(TestCase):
         """
         log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 1", str(log5))
-        self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertIn(
+            "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught",
+            str(log5),
+        )
         self.assertIn(
             "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
-            str(log5))
-        self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+            str(log5),
+        )
+        self.assertNotIn(
+            "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught",
+            str(log5),
+        )
         self.assertNotIn(
             "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
-            str(log5))
-        item_recovered = {'processed': [
-            'NotGeneratorRecoverMiddleware.process_spider_exception',
-            'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+            str(log5),
+        )
+        item_recovered = {
+            "processed": [
+                "NotGeneratorRecoverMiddleware.process_spider_exception",
+                "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
+            ]
+        }
         self.assertIn(str(item_recovered), str(log5))
-        self.assertNotIn('parse-first-item', str(log5))
-        self.assertNotIn('parse-second-item', str(log5))
+        self.assertNotIn("parse-first-item", str(log5))
+        self.assertNotIn("parse-second-item", str(log5))
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 63daf0b8a..db5a22adb 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -36,11 +36,11 @@ class TestRefererMiddleware(TestCase):
     resp_headers = {}
     settings = {}
     scenarii = [
-        ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),
+        ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
     ]
 
     def setUp(self):
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
         settings = Settings(self.settings)
         self.mw = RefererMiddleware(settings)
 
@@ -56,7 +56,7 @@ class TestRefererMiddleware(TestCase):
             response = self.get_response(origin)
             request = self.get_request(target)
             out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEqual(out[0].headers.get('Referer'), referrer)
+            self.assertEqual(out[0].headers.get("Referer"), referrer)
 
 
 class MixinDefault:
@@ -65,101 +65,184 @@ class MixinDefault:
 
     with some additional filtering of s3://
     """
+
     scenarii = [
-        ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
-        ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
-        ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
-        ('https://example.com/', 'http://scrapy.org/', None),
-
+        ("https://example.com/", "https://scrapy.org/", b"https://example.com/"),
+        ("http://example.com/", "http://scrapy.org/", b"http://example.com/"),
+        ("http://example.com/", "https://scrapy.org/", b"http://example.com/"),
+        ("https://example.com/", "http://scrapy.org/", None),
         # no credentials leak
-        ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
-
+        (
+            "http://user:password@example.com/",
+            "https://scrapy.org/",
+            b"http://example.com/",
+        ),
         # no referrer leak for local schemes
-        ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
-        ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
-
+        ("file:///home/path/to/somefile.html", "https://scrapy.org/", None),
+        ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),
         # no referrer leak for s3 origins
-        ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
-        ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
+        ("s3://mybucket/path/to/data.csv", "https://scrapy.org/", None),
+        ("s3://mybucket/path/to/data.csv", "http://scrapy.org/", None),
     ]
 
 
 class MixinNoReferrer:
     scenarii = [
-        ('https://example.com/page.html', 'https://example.com/', None),
-        ('http://www.example.com/', 'https://scrapy.org/', None),
-        ('http://www.example.com/', 'http://scrapy.org/', None),
-        ('https://www.example.com/', 'http://scrapy.org/', None),
-        ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
+        ("https://example.com/page.html", "https://example.com/", None),
+        ("http://www.example.com/", "https://scrapy.org/", None),
+        ("http://www.example.com/", "http://scrapy.org/", None),
+        ("https://www.example.com/", "http://scrapy.org/", None),
+        ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),
     ]
 
 
 class MixinNoReferrerWhenDowngrade:
     scenarii = [
         # TLS to TLS: send non-empty referrer
-        ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
-        ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
-        ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
-        ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
-        ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
-
+        (
+            "https://example.com/page.html",
+            "https://not.example.com/",
+            b"https://example.com/page.html",
+        ),
+        (
+            "https://example.com/page.html",
+            "https://scrapy.org/",
+            b"https://example.com/page.html",
+        ),
+        (
+            "https://example.com:443/page.html",
+            "https://scrapy.org/",
+            b"https://example.com/page.html",
+        ),
+        (
+            "https://example.com:444/page.html",
+            "https://scrapy.org/",
+            b"https://example.com:444/page.html",
+        ),
+        (
+            "ftps://example.com/urls.zip",
+            "https://scrapy.org/",
+            b"ftps://example.com/urls.zip",
+        ),
         # TLS to non-TLS: do not send referrer
-        ('https://example.com/page.html', 'http://not.example.com/', None),
-        ('https://example.com/page.html', 'http://scrapy.org/', None),
-        ('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
-
+        ("https://example.com/page.html", "http://not.example.com/", None),
+        ("https://example.com/page.html", "http://scrapy.org/", None),
+        ("ftps://example.com/urls.zip", "http://scrapy.org/", None),
         # non-TLS to TLS or non-TLS: send referrer
-        ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
-        ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
-        ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
-        ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
-        ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
-        ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
-        ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
-        ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
-
+        (
+            "http://example.com/page.html",
+            "https://not.example.com/",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "https://scrapy.org/",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com:8080/page.html",
+            "https://scrapy.org/",
+            b"http://example.com:8080/page.html",
+        ),
+        (
+            "http://example.com:80/page.html",
+            "http://not.example.com/",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "http://scrapy.org/",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com:443/page.html",
+            "http://scrapy.org/",
+            b"http://example.com:443/page.html",
+        ),
+        (
+            "ftp://example.com/urls.zip",
+            "http://scrapy.org/",
+            b"ftp://example.com/urls.zip",
+        ),
+        (
+            "ftp://example.com/urls.zip",
+            "https://scrapy.org/",
+            b"ftp://example.com/urls.zip",
+        ),
         # test for user/password stripping
-        ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
+        (
+            "http://user:password@example.com/page.html",
+            "https://not.example.com/",
+            b"http://example.com/page.html",
+        ),
     ]
 
 
 class MixinSameOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
-        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
-        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
-        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
-        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
         (
-            'http://example.com:8888/page.html',
-            'http://example.com:8888/not-page.html',
-            b'http://example.com:8888/page.html',
+            "https://example.com/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
         ),
-
-        # Different host: do NOT send referrer
-        ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
-        ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
-        ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
-
-        # Different port: do NOT send referrer
-        ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
-        ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
-        ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
-
-        # Different protocols: do NOT send refferer
-        ('https://example.com/page.html', 'http://example.com/not-page.html', None),
-        ('https://example.com/page.html', 'http://not.example.com/', None),
-        ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
-        ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
-        ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
-
-        # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
         (
-            'https://user:password@example.com/page.html',
-            'https://example.com/not-page.html',
-            b'https://example.com/page.html',
+            "http://example.com/page.html",
+            "http://example.com/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "https://example.com:443/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
+        ),
+        (
+            "http://example.com:80/page.html",
+            "http://example.com/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "http://example.com:80/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com:8888/page.html",
+            "http://example.com:8888/not-page.html",
+            b"http://example.com:8888/page.html",
+        ),
+        # Different host: do NOT send referrer
+        (
+            "https://example.com/page.html",
+            "https://not.example.com/otherpage.html",
+            None,
+        ),
+        ("http://example.com/page.html", "http://not.example.com/otherpage.html", None),
+        ("http://example.com/page.html", "http://www.example.com/otherpage.html", None),
+        # Different port: do NOT send referrer
+        (
+            "https://example.com:444/page.html",
+            "https://example.com/not-page.html",
+            None,
+        ),
+        ("http://example.com:81/page.html", "http://example.com/not-page.html", None),
+        ("http://example.com/page.html", "http://example.com:81/not-page.html", None),
+        # Different protocols: do NOT send refferer
+        ("https://example.com/page.html", "http://example.com/not-page.html", None),
+        ("https://example.com/page.html", "http://not.example.com/", None),
+        ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+        ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),
+        ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+        # test for user/password stripping
+        (
+            "https://user:password@example.com/page.html",
+            "http://example.com/not-page.html",
+            None,
+        ),
+        (
+            "https://user:password@example.com/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
         ),
     ]
 
@@ -167,78 +250,158 @@ class MixinSameOrigin:
 class MixinOrigin:
     scenarii = [
         # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
-        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
-        ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
-        ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
-        ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
-
+        (
+            "https://example.com/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/",
+        ),
+        (
+            "https://example.com/page.html",
+            "https://scrapy.org",
+            b"https://example.com/",
+        ),
+        ("https://example.com/page.html", "http://scrapy.org", b"https://example.com/"),
+        ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),
         # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
+        (
+            "https://user:password@example.com/page.html",
+            "http://scrapy.org",
+            b"https://example.com/",
+        ),
     ]
 
 
 class MixinStrictOrigin:
     scenarii = [
         # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
-        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
-        ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
-        ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
-
+        (
+            "https://example.com/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/",
+        ),
+        (
+            "https://example.com/page.html",
+            "https://scrapy.org",
+            b"https://example.com/",
+        ),
+        ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),
         # downgrade: send nothing
-        ('https://example.com/page.html', 'http://scrapy.org', None),
-
+        ("https://example.com/page.html", "http://scrapy.org", None),
         # upgrade: send origin
-        ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),
-
+        ("http://example.com/page.html", "https://scrapy.org", b"http://example.com/"),
         # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
-        ('https://user:password@example.com/page.html', 'http://scrapy.org', None),
+        (
+            "https://user:password@example.com/page.html",
+            "https://scrapy.org",
+            b"https://example.com/",
+        ),
+        ("https://user:password@example.com/page.html", "http://scrapy.org", None),
     ]
 
 
 class MixinOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
-        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
-        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
-        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
-        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
         (
-            'http://example.com:8888/page.html',
-            'http://example.com:8888/not-page.html',
-            b'http://example.com:8888/page.html',
+            "https://example.com/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "http://example.com/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "https://example.com:443/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
+        ),
+        (
+            "http://example.com:80/page.html",
+            "http://example.com/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "http://example.com:80/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com:8888/page.html",
+            "http://example.com:8888/not-page.html",
+            b"http://example.com:8888/page.html",
         ),
-
         # Different host: send origin as referrer
-        ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
-        ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
-        ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
+        (
+            "https://example2.com/page.html",
+            "https://scrapy.org/otherpage.html",
+            b"https://example2.com/",
+        ),
+        (
+            "https://example2.com/page.html",
+            "https://not.example2.com/otherpage.html",
+            b"https://example2.com/",
+        ),
+        (
+            "http://example2.com/page.html",
+            "http://not.example2.com/otherpage.html",
+            b"http://example2.com/",
+        ),
         # exact match required
-        ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
-
+        (
+            "http://example2.com/page.html",
+            "http://www.example2.com/otherpage.html",
+            b"http://example2.com/",
+        ),
         # Different port: send origin as referrer
-        ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
-        ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
-
+        (
+            "https://example3.com:444/page.html",
+            "https://example3.com/not-page.html",
+            b"https://example3.com:444/",
+        ),
+        (
+            "http://example3.com:81/page.html",
+            "http://example3.com/not-page.html",
+            b"http://example3.com:81/",
+        ),
         # Different protocols: send origin as referrer
-        ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
-        ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
-        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
-        ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
-        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
-
+        (
+            "https://example4.com/page.html",
+            "http://example4.com/not-page.html",
+            b"https://example4.com/",
+        ),
+        (
+            "https://example4.com/page.html",
+            "http://not.example4.com/",
+            b"https://example4.com/",
+        ),
+        (
+            "ftps://example4.com/urls.zip",
+            "https://example4.com/not-page.html",
+            b"ftps://example4.com/",
+        ),
+        (
+            "ftp://example4.com/urls.zip",
+            "http://example4.com/not-page.html",
+            b"ftp://example4.com/",
+        ),
+        (
+            "ftps://example4.com/urls.zip",
+            "https://example4.com/not-page.html",
+            b"ftps://example4.com/",
+        ),
         # test for user/password stripping
         (
-            'https://user:password@example5.com/page.html',
-            'https://example5.com/not-page.html',
-            b'https://example5.com/page.html',
+            "https://user:password@example5.com/page.html",
+            "https://example5.com/not-page.html",
+            b"https://example5.com/page.html",
         ),
         # TLS to non-TLS downgrade: send origin
         (
-            'https://user:password@example5.com/page.html',
-            'http://example5.com/not-page.html',
-            b'https://example5.com/',
+            "https://user:password@example5.com/page.html",
+            "http://example5.com/not-page.html",
+            b"https://example5.com/",
         ),
     ]
 
@@ -246,90 +409,215 @@ class MixinOriginWhenCrossOrigin:
 class MixinStrictOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
-        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
-        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
-        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
-        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
         (
-            'http://example.com:8888/page.html',
-            'http://example.com:8888/not-page.html',
-            b'http://example.com:8888/page.html',
+            "https://example.com/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "http://example.com/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "https://example.com:443/page.html",
+            "https://example.com/not-page.html",
+            b"https://example.com/page.html",
+        ),
+        (
+            "http://example.com:80/page.html",
+            "http://example.com/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com/page.html",
+            "http://example.com:80/not-page.html",
+            b"http://example.com/page.html",
+        ),
+        (
+            "http://example.com:8888/page.html",
+            "http://example.com:8888/not-page.html",
+            b"http://example.com:8888/page.html",
         ),
-
         # Different host: send origin as referrer
-        ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
-        ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
-        ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
+        (
+            "https://example2.com/page.html",
+            "https://scrapy.org/otherpage.html",
+            b"https://example2.com/",
+        ),
+        (
+            "https://example2.com/page.html",
+            "https://not.example2.com/otherpage.html",
+            b"https://example2.com/",
+        ),
+        (
+            "http://example2.com/page.html",
+            "http://not.example2.com/otherpage.html",
+            b"http://example2.com/",
+        ),
         # exact match required
-        ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
-
+        (
+            "http://example2.com/page.html",
+            "http://www.example2.com/otherpage.html",
+            b"http://example2.com/",
+        ),
         # Different port: send origin as referrer
-        ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
-        ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
-
+        (
+            "https://example3.com:444/page.html",
+            "https://example3.com/not-page.html",
+            b"https://example3.com:444/",
+        ),
+        (
+            "http://example3.com:81/page.html",
+            "http://example3.com/not-page.html",
+            b"http://example3.com:81/",
+        ),
         # downgrade
-        ('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
-        ('https://example4.com/page.html', 'http://not.example4.com/', None),
-
+        ("https://example4.com/page.html", "http://example4.com/not-page.html", None),
+        ("https://example4.com/page.html", "http://not.example4.com/", None),
         # non-TLS to non-TLS
-        ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
-
+        (
+            "ftp://example4.com/urls.zip",
+            "http://example4.com/not-page.html",
+            b"ftp://example4.com/",
+        ),
         # upgrade
-        ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
-        ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
-
+        (
+            "http://example4.com/page.html",
+            "https://example4.com/not-page.html",
+            b"http://example4.com/",
+        ),
+        (
+            "http://example4.com/page.html",
+            "https://not.example4.com/",
+            b"http://example4.com/",
+        ),
         # Different protocols: send origin as referrer
-        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
-        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
-
+        (
+            "ftps://example4.com/urls.zip",
+            "https://example4.com/not-page.html",
+            b"ftps://example4.com/",
+        ),
+        (
+            "ftps://example4.com/urls.zip",
+            "https://example4.com/not-page.html",
+            b"ftps://example4.com/",
+        ),
         # test for user/password stripping
         (
-            'https://user:password@example5.com/page.html',
-            'https://example5.com/not-page.html',
-            b'https://example5.com/page.html',
+            "https://user:password@example5.com/page.html",
+            "https://example5.com/not-page.html",
+            b"https://example5.com/page.html",
         ),
-
         # TLS to non-TLS downgrade: send nothing
-        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
+        (
+            "https://user:password@example5.com/page.html",
+            "http://example5.com/not-page.html",
+            None,
+        ),
     ]
 
 
 class MixinUnsafeUrl:
     scenarii = [
         # TLS to TLS: send referrer
-        ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
-        ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
-        ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
-        ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
-        ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
-        ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
-
-        # TLS to non-TLS: send referrer (yes, it's unsafe)
-        ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
-        ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
-        ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
-
-        # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
-        ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
-        ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
-        ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
-        ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
-        ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
-        ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
-        ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
-        ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
-
-        # test for user/password stripping
         (
-            'http://user:password@example4.com/page.html',
-            'https://not.example4.com/',
-            b'http://example4.com/page.html',
+            "https://example.com/sekrit.html",
+            "http://not.example.com/",
+            b"https://example.com/sekrit.html",
         ),
         (
-            'https://user:password@example4.com/page.html',
-            'http://scrapy.org/',
-            b'https://example4.com/page.html',
+            "https://example1.com/page.html",
+            "https://not.example1.com/",
+            b"https://example1.com/page.html",
+        ),
+        (
+            "https://example1.com/page.html",
+            "https://scrapy.org/",
+            b"https://example1.com/page.html",
+        ),
+        (
+            "https://example1.com:443/page.html",
+            "https://scrapy.org/",
+            b"https://example1.com/page.html",
+        ),
+        (
+            "https://example1.com:444/page.html",
+            "https://scrapy.org/",
+            b"https://example1.com:444/page.html",
+        ),
+        (
+            "ftps://example1.com/urls.zip",
+            "https://scrapy.org/",
+            b"ftps://example1.com/urls.zip",
+        ),
+        # TLS to non-TLS: send referrer (yes, it's unsafe)
+        (
+            "https://example2.com/page.html",
+            "http://not.example2.com/",
+            b"https://example2.com/page.html",
+        ),
+        (
+            "https://example2.com/page.html",
+            "http://scrapy.org/",
+            b"https://example2.com/page.html",
+        ),
+        (
+            "ftps://example2.com/urls.zip",
+            "http://scrapy.org/",
+            b"ftps://example2.com/urls.zip",
+        ),
+        # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
+        (
+            "http://example3.com/page.html",
+            "https://not.example3.com/",
+            b"http://example3.com/page.html",
+        ),
+        (
+            "http://example3.com/page.html",
+            "https://scrapy.org/",
+            b"http://example3.com/page.html",
+        ),
+        (
+            "http://example3.com:8080/page.html",
+            "https://scrapy.org/",
+            b"http://example3.com:8080/page.html",
+        ),
+        (
+            "http://example3.com:80/page.html",
+            "http://not.example3.com/",
+            b"http://example3.com/page.html",
+        ),
+        (
+            "http://example3.com/page.html",
+            "http://scrapy.org/",
+            b"http://example3.com/page.html",
+        ),
+        (
+            "http://example3.com:443/page.html",
+            "http://scrapy.org/",
+            b"http://example3.com:443/page.html",
+        ),
+        (
+            "ftp://example3.com/urls.zip",
+            "http://scrapy.org/",
+            b"ftp://example3.com/urls.zip",
+        ),
+        (
+            "ftp://example3.com/urls.zip",
+            "https://scrapy.org/",
+            b"ftp://example3.com/urls.zip",
+        ),
+        # test for user/password stripping
+        (
+            "http://user:password@example4.com/page.html",
+            "https://not.example4.com/",
+            b"http://example4.com/page.html",
+        ),
+        (
+            "https://user:password@example4.com/page.html",
+            "http://scrapy.org/",
+            b"https://example4.com/page.html",
         ),
     ]
 
@@ -340,35 +628,49 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
 
 # --- Tests using settings to set policy using class path
 class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerPolicy"}
 
 
-class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+class TestSettingsNoReferrerWhenDowngrade(
+    MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
+    }
 
 
 class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
 
 
 class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginPolicy"}
 
 
 class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginPolicy"
+    }
 
 
-class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+class TestSettingsOriginWhenCrossOrigin(
+    MixinOriginWhenCrossOrigin, TestRefererMiddleware
+):
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+    }
 
 
-class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
+class TestSettingsStrictOriginWhenCrossOrigin(
+    MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware
+):
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy"
+    }
 
 
 class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
 
 
 class CustomPythonOrgPolicy(ReferrerPolicy):
@@ -376,77 +678,95 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
     A dummy policy that returns referrer as http(s)://python.org
     depending on the scheme of the target URL.
     """
+
     def referrer(self, response, request):
         scheme = urlparse(request).scheme
-        if scheme == 'https':
-            return b'https://python.org/'
-        if scheme == 'http':
-            return b'http://python.org/'
+        if scheme == "https":
+            return b"https://python.org/"
+        if scheme == "http":
+            return b"http://python.org/"
 
 
 class TestSettingsCustomPolicy(TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': CustomPythonOrgPolicy}
+    settings = {"REFERRER_POLICY": CustomPythonOrgPolicy}
     scenarii = [
-        ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
-        ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
-        ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'),
-        ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'),
-        ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'),
-        ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'),
-
+        ("https://example.com/", "https://scrapy.org/", b"https://python.org/"),
+        ("http://example.com/", "http://scrapy.org/", b"http://python.org/"),
+        ("http://example.com/", "https://scrapy.org/", b"https://python.org/"),
+        ("https://example.com/", "http://scrapy.org/", b"http://python.org/"),
+        (
+            "file:///home/path/to/somefile.html",
+            "https://scrapy.org/",
+            b"https://python.org/",
+        ),
+        (
+            "file:///home/path/to/somefile.html",
+            "http://scrapy.org/",
+            b"http://python.org/",
+        ),
     ]
 
 
 # --- Tests using Request meta dict to set policy
 class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
+    req_meta = {"referrer_policy": POLICY_SCRAPY_DEFAULT}
 
 
 class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+    req_meta = {"referrer_policy": POLICY_NO_REFERRER}
 
 
-class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}
+class TestRequestMetaNoReferrerWhenDowngrade(
+    MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
+    req_meta = {"referrer_policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE}
 
 
 class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}
+    req_meta = {"referrer_policy": POLICY_SAME_ORIGIN}
 
 
 class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_ORIGIN}
+    req_meta = {"referrer_policy": POLICY_ORIGIN}
 
 
 class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}
+    req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN}
 
 
-class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+class TestRequestMetaOriginWhenCrossOrigin(
+    MixinOriginWhenCrossOrigin, TestRefererMiddleware
+):
+    req_meta = {"referrer_policy": POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
 
 
-class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+class TestRequestMetaStrictOriginWhenCrossOrigin(
+    MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware
+):
+    req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
 
 
 class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
-    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+    req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
 
 
 class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
-    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
+    req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
 
 
 class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
-    req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
+    }
+    req_meta = {"referrer_policy": POLICY_NO_REFERRER}
 
 
 class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
-    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+    }
+    req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
 
 
 class TestRequestMetaSettingFallback(TestCase):
@@ -456,57 +776,65 @@ class TestRequestMetaSettingFallback(TestCase):
             # When an unknown policy is referenced in Request.meta
             # (here, a typo error),
             # the policy defined in settings takes precedence
-            {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'},
+            {
+                "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+            },
             {},
-            {'referrer_policy': 'ssscrapy-default'},
+            {"referrer_policy": "ssscrapy-default"},
             OriginWhenCrossOriginPolicy,
-            True
+            True,
         ),
         (
             # same as above but with string value for settings policy
-            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {"REFERRER_POLICY": "origin-when-cross-origin"},
             {},
-            {'referrer_policy': 'ssscrapy-default'},
+            {"referrer_policy": "ssscrapy-default"},
             OriginWhenCrossOriginPolicy,
-            True
+            True,
         ),
         (
             # request meta references a wrong policy but it is set,
             # so the Referrer-Policy header in response is not used,
             # and the settings' policy is applied
-            {'REFERRER_POLICY': 'origin-when-cross-origin'},
-            {'Referrer-Policy': 'unsafe-url'},
-            {'referrer_policy': 'ssscrapy-default'},
+            {"REFERRER_POLICY": "origin-when-cross-origin"},
+            {"Referrer-Policy": "unsafe-url"},
+            {"referrer_policy": "ssscrapy-default"},
             OriginWhenCrossOriginPolicy,
-            True
+            True,
         ),
         (
             # here, request meta does not set the policy
             # so response headers take precedence
-            {'REFERRER_POLICY': 'origin-when-cross-origin'},
-            {'Referrer-Policy': 'unsafe-url'},
+            {"REFERRER_POLICY": "origin-when-cross-origin"},
+            {"Referrer-Policy": "unsafe-url"},
             {},
             UnsafeUrlPolicy,
-            False
+            False,
         ),
         (
             # here, request meta does not set the policy,
             # but response headers also use an unknown policy,
             # so the settings' policy is used
-            {'REFERRER_POLICY': 'origin-when-cross-origin'},
-            {'Referrer-Policy': 'unknown'},
+            {"REFERRER_POLICY": "origin-when-cross-origin"},
+            {"Referrer-Policy": "unknown"},
             {},
             OriginWhenCrossOriginPolicy,
-            True
-        )
+            True,
+        ),
     ]
 
     def test(self):
 
-        origin = 'http://www.scrapy.org'
-        target = 'http://www.example.com'
+        origin = "http://www.scrapy.org"
+        target = "http://www.example.com"
 
-        for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
+        for (
+            settings,
+            response_headers,
+            request_meta,
+            policy_class,
+            check_warning,
+        ) in self.params[3:]:
             mw = RefererMiddleware(Settings(settings))
 
             response = Response(origin, headers=response_headers)
@@ -522,7 +850,6 @@ class TestRequestMetaSettingFallback(TestCase):
 
 
 class TestSettingsPolicyByName(TestCase):
-
     def test_valid_name(self):
         for s, p in [
             (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
@@ -535,7 +862,7 @@ class TestSettingsPolicyByName(TestCase):
             (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
             (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
         ]:
-            settings = Settings({'REFERRER_POLICY': s})
+            settings = Settings({"REFERRER_POLICY": s})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
 
@@ -551,98 +878,121 @@ class TestSettingsPolicyByName(TestCase):
             (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
             (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
         ]:
-            settings = Settings({'REFERRER_POLICY': s.upper()})
+            settings = Settings({"REFERRER_POLICY": s.upper()})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
 
     def test_invalid_name(self):
-        settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
+        settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
         with self.assertRaises(RuntimeError):
             RefererMiddleware(settings)
 
 
 class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
-    resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
+    resp_headers = {"Referrer-Policy": POLICY_UNSAFE_URL.upper()}
 
 
 class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
-    resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
+    }
+    resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER.swapcase()}
 
 
-class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
-    resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
+class TestPolicyHeaderPredecence003(
+    MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+    }
+    resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
 
-class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestPolicyHeaderPredecence004(
+    MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
     """
     The empty string means "no-referrer-when-downgrade"
     """
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
-    resp_headers = {'Referrer-Policy': ''}
+
+    settings = {
+        "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+    }
+    resp_headers = {"Referrer-Policy": ""}
 
 
 class TestReferrerOnRedirect(TestRefererMiddleware):
 
-    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+    settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
     scenarii = [
         (
-            'http://scrapytest.org/1',      # parent
-            'http://scrapytest.org/2',      # target
+            "http://scrapytest.org/1",  # parent
+            "http://scrapytest.org/2",  # target
             (
                 # redirections: code, URL
-                (301, 'http://scrapytest.org/3'),
-                (301, 'http://scrapytest.org/4'),
+                (301, "http://scrapytest.org/3"),
+                (301, "http://scrapytest.org/4"),
             ),
-            b'http://scrapytest.org/1',  # expected initial referer
-            b'http://scrapytest.org/1',  # expected referer for the redirection request
+            b"http://scrapytest.org/1",  # expected initial referer
+            b"http://scrapytest.org/1",  # expected referer for the redirection request
         ),
         (
-            'https://scrapytest.org/1',
-            'https://scrapytest.org/2',
+            "https://scrapytest.org/1",
+            "https://scrapytest.org/2",
             (
                 # redirecting to non-secure URL
-                (301, 'http://scrapytest.org/3'),
+                (301, "http://scrapytest.org/3"),
             ),
-            b'https://scrapytest.org/1',
-            b'https://scrapytest.org/1',
+            b"https://scrapytest.org/1",
+            b"https://scrapytest.org/1",
         ),
         (
-            'https://scrapytest.org/1',
-            'https://scrapytest.com/2',
+            "https://scrapytest.org/1",
+            "https://scrapytest.com/2",
             (
                 # redirecting to non-secure URL: different origin
-                (301, 'http://scrapytest.com/3'),
+                (301, "http://scrapytest.com/3"),
             ),
-            b'https://scrapytest.org/1',
-            b'https://scrapytest.org/1',
+            b"https://scrapytest.org/1",
+            b"https://scrapytest.org/1",
         ),
     ]
 
     def setUp(self):
-        self.spider = Spider('foo')
+        self.spider = Spider("foo")
         settings = Settings(self.settings)
         self.referrermw = RefererMiddleware(settings)
         self.redirectmw = RedirectMiddleware(settings)
 
     def test(self):
 
-        for parent, target, redirections, init_referrer, final_referrer in self.scenarii:
+        for (
+            parent,
+            target,
+            redirections,
+            init_referrer,
+            final_referrer,
+        ) in self.scenarii:
             response = self.get_response(parent)
             request = self.get_request(target)
 
-            out = list(self.referrermw.process_spider_output(response, [request], self.spider))
-            self.assertEqual(out[0].headers.get('Referer'), init_referrer)
+            out = list(
+                self.referrermw.process_spider_output(response, [request], self.spider)
+            )
+            self.assertEqual(out[0].headers.get("Referer"), init_referrer)
 
             for status, url in redirections:
-                response = Response(request.url, headers={'Location': url}, status=status)
-                request = self.redirectmw.process_response(request, response, self.spider)
+                response = Response(
+                    request.url, headers={"Location": url}, status=status
+                )
+                request = self.redirectmw.process_response(
+                    request, response, self.spider
+                )
                 self.referrermw.request_scheduled(request, self.spider)
 
             assert isinstance(request, Request)
-            self.assertEqual(request.headers.get('Referer'), final_referrer)
+            self.assertEqual(request.headers.get("Referer"), final_referrer)
 
 
 class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
@@ -650,34 +1000,31 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
     No Referrer policy never sets the "Referer" header.
     HTTP redirections should not change that.
     """
-    settings = {'REFERRER_POLICY': 'no-referrer'}
+
+    settings = {"REFERRER_POLICY": "no-referrer"}
     scenarii = [
         (
-            'http://scrapytest.org/1',      # parent
-            'http://scrapytest.org/2',      # target
+            "http://scrapytest.org/1",  # parent
+            "http://scrapytest.org/2",  # target
             (
                 # redirections: code, URL
-                (301, 'http://scrapytest.org/3'),
-                (301, 'http://scrapytest.org/4'),
+                (301, "http://scrapytest.org/3"),
+                (301, "http://scrapytest.org/4"),
             ),
             None,  # expected initial "Referer"
             None,  # expected "Referer" for the redirection request
         ),
         (
-            'https://scrapytest.org/1',
-            'https://scrapytest.org/2',
-            (
-                (301, 'http://scrapytest.org/3'),
-            ),
+            "https://scrapytest.org/1",
+            "https://scrapytest.org/2",
+            ((301, "http://scrapytest.org/3"),),
             None,
             None,
         ),
         (
-            'https://scrapytest.org/1',
-            'https://example.com/2',    # different origin
-            (
-                (301, 'http://scrapytest.com/3'),
-            ),
+            "https://scrapytest.org/1",
+            "https://example.com/2",  # different origin
+            ((301, "http://scrapytest.com/3"),),
             None,
             None,
         ),
@@ -692,37 +1039,38 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
     HTTP redirections to a different domain or a lower secure level
     should have the "Referer" removed.
     """
-    settings = {'REFERRER_POLICY': 'same-origin'}
+
+    settings = {"REFERRER_POLICY": "same-origin"}
     scenarii = [
         (
-            'http://scrapytest.org/101',      # origin
-            'http://scrapytest.org/102',      # target
+            "http://scrapytest.org/101",  # origin
+            "http://scrapytest.org/102",  # target
             (
                 # redirections: code, URL
-                (301, 'http://scrapytest.org/103'),
-                (301, 'http://scrapytest.org/104'),
+                (301, "http://scrapytest.org/103"),
+                (301, "http://scrapytest.org/104"),
             ),
-            b'http://scrapytest.org/101',  # expected initial "Referer"
-            b'http://scrapytest.org/101',  # expected referer for the redirection request
+            b"http://scrapytest.org/101",  # expected initial "Referer"
+            b"http://scrapytest.org/101",  # expected referer for the redirection request
         ),
         (
-            'https://scrapytest.org/201',
-            'https://scrapytest.org/202',
+            "https://scrapytest.org/201",
+            "https://scrapytest.org/202",
             (
                 # redirecting from secure to non-secure URL == different origin
-                (301, 'http://scrapytest.org/203'),
+                (301, "http://scrapytest.org/203"),
             ),
-            b'https://scrapytest.org/201',
+            b"https://scrapytest.org/201",
             None,
         ),
         (
-            'https://scrapytest.org/301',
-            'https://scrapytest.org/302',
+            "https://scrapytest.org/301",
+            "https://scrapytest.org/302",
             (
                 # different domain == different origin
-                (301, 'http://example.com/303'),
+                (301, "http://example.com/303"),
             ),
-            b'https://scrapytest.org/301',
+            b"https://scrapytest.org/301",
             None,
         ),
     ]
@@ -737,63 +1085,65 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
     Redirections from secure to non-secure URLs should have the
     "Referrer" header removed if necessary.
     """
-    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
+
+    settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN}
     scenarii = [
         (
-            'http://scrapytest.org/101',
-            'http://scrapytest.org/102',
+            "http://scrapytest.org/101",
+            "http://scrapytest.org/102",
             (
-                (301, 'http://scrapytest.org/103'),
-                (301, 'http://scrapytest.org/104'),
+                (301, "http://scrapytest.org/103"),
+                (301, "http://scrapytest.org/104"),
             ),
-            b'http://scrapytest.org/',  # send origin
-            b'http://scrapytest.org/',  # redirects to same origin: send origin
+            b"http://scrapytest.org/",  # send origin
+            b"http://scrapytest.org/",  # redirects to same origin: send origin
         ),
         (
-            'https://scrapytest.org/201',
-            'https://scrapytest.org/202',
+            "https://scrapytest.org/201",
+            "https://scrapytest.org/202",
             (
                 # redirecting to non-secure URL: no referrer
-                (301, 'http://scrapytest.org/203'),
+                (301, "http://scrapytest.org/203"),
             ),
-            b'https://scrapytest.org/',
+            b"https://scrapytest.org/",
             None,
         ),
         (
-            'https://scrapytest.org/301',
-            'https://scrapytest.org/302',
+            "https://scrapytest.org/301",
+            "https://scrapytest.org/302",
             (
                 # redirecting to non-secure URL (different domain): no referrer
-                (301, 'http://example.com/303'),
+                (301, "http://example.com/303"),
             ),
-            b'https://scrapytest.org/',
+            b"https://scrapytest.org/",
             None,
         ),
         (
-            'http://scrapy.org/401',
-            'http://example.com/402',
-            (
-                (301, 'http://scrapytest.org/403'),
-            ),
-            b'http://scrapy.org/',
-            b'http://scrapy.org/',
+            "http://scrapy.org/401",
+            "http://example.com/402",
+            ((301, "http://scrapytest.org/403"),),
+            b"http://scrapy.org/",
+            b"http://scrapy.org/",
         ),
         (
-            'https://scrapy.org/501',
-            'https://example.com/502',
+            "https://scrapy.org/501",
+            "https://example.com/502",
             (
                 # HTTPS all along, so origin referrer is kept as-is
-                (301, 'https://google.com/503'),
-                (301, 'https://facebook.com/504'),
+                (301, "https://google.com/503"),
+                (301, "https://facebook.com/504"),
             ),
-            b'https://scrapy.org/',
-            b'https://scrapy.org/',
+            b"https://scrapy.org/",
+            b"https://scrapy.org/",
         ),
         (
-            'https://scrapytest.org/601',
-            'http://scrapytest.org/602',                # TLS to non-TLS: no referrer
+            "https://scrapytest.org/601",
+            "http://scrapytest.org/602",  # TLS to non-TLS: no referrer
             (
-                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) no referrer
+                (
+                    301,
+                    "https://scrapytest.org/603",
+                ),  # TLS URL again: (still) no referrer
             ),
             None,
             None,
@@ -810,67 +1160,64 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
     Redirections to a different origin should strip the "Referer"
     to the parent origin.
     """
-    settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+
+    settings = {"REFERRER_POLICY": POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
     scenarii = [
         (
-            'http://scrapytest.org/101',      # origin
-            'http://scrapytest.org/102',      # target + redirection
+            "http://scrapytest.org/101",  # origin
+            "http://scrapytest.org/102",  # target + redirection
             (
                 # redirections: code, URL
-                (301, 'http://scrapytest.org/103'),
-                (301, 'http://scrapytest.org/104'),
+                (301, "http://scrapytest.org/103"),
+                (301, "http://scrapytest.org/104"),
             ),
-            b'http://scrapytest.org/101',  # expected initial referer
-            b'http://scrapytest.org/101',  # expected referer for the redirection request
+            b"http://scrapytest.org/101",  # expected initial referer
+            b"http://scrapytest.org/101",  # expected referer for the redirection request
         ),
         (
-            'https://scrapytest.org/201',
-            'https://scrapytest.org/202',
+            "https://scrapytest.org/201",
+            "https://scrapytest.org/202",
             (
                 # redirecting to non-secure URL: send origin
-                (301, 'http://scrapytest.org/203'),
+                (301, "http://scrapytest.org/203"),
             ),
-            b'https://scrapytest.org/201',
-            b'https://scrapytest.org/',
+            b"https://scrapytest.org/201",
+            b"https://scrapytest.org/",
         ),
         (
-            'https://scrapytest.org/301',
-            'https://scrapytest.org/302',
+            "https://scrapytest.org/301",
+            "https://scrapytest.org/302",
             (
                 # redirecting to non-secure URL (different domain): send origin
-                (301, 'http://example.com/303'),
+                (301, "http://example.com/303"),
             ),
-            b'https://scrapytest.org/301',
-            b'https://scrapytest.org/',
+            b"https://scrapytest.org/301",
+            b"https://scrapytest.org/",
         ),
         (
-            'http://scrapy.org/401',
-            'http://example.com/402',
-            (
-                (301, 'http://scrapytest.org/403'),
-            ),
-            b'http://scrapy.org/',
-            b'http://scrapy.org/',
+            "http://scrapy.org/401",
+            "http://example.com/402",
+            ((301, "http://scrapytest.org/403"),),
+            b"http://scrapy.org/",
+            b"http://scrapy.org/",
         ),
         (
-            'https://scrapy.org/501',
-            'https://example.com/502',
+            "https://scrapy.org/501",
+            "https://example.com/502",
             (
                 # all different domains: send origin
-                (301, 'https://google.com/503'),
-                (301, 'https://facebook.com/504'),
+                (301, "https://google.com/503"),
+                (301, "https://facebook.com/504"),
             ),
-            b'https://scrapy.org/',
-            b'https://scrapy.org/',
+            b"https://scrapy.org/",
+            b"https://scrapy.org/",
         ),
         (
-            'https://scrapytest.org/301',
-            'http://scrapytest.org/302',                # TLS to non-TLS: send origin
-            (
-                (301, 'https://scrapytest.org/303'),    # TLS URL again: send origin (also)
-            ),
-            b'https://scrapytest.org/',
-            b'https://scrapytest.org/',
+            "https://scrapytest.org/301",
+            "http://scrapytest.org/302",  # TLS to non-TLS: send origin
+            ((301, "https://scrapytest.org/303"),),  # TLS URL again: send origin (also)
+            b"https://scrapytest.org/",
+            b"https://scrapytest.org/",
         ),
     ]
 
@@ -886,64 +1233,66 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
     Redirections to a different origin should strip the "Referer" to the parent origin,
     and from https:// to http:// will remove the "Referer" header.
     """
-    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+
+    settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
     scenarii = [
         (
-            'http://scrapytest.org/101',      # origin
-            'http://scrapytest.org/102',      # target + redirection
+            "http://scrapytest.org/101",  # origin
+            "http://scrapytest.org/102",  # target + redirection
             (
                 # redirections: code, URL
-                (301, 'http://scrapytest.org/103'),
-                (301, 'http://scrapytest.org/104'),
+                (301, "http://scrapytest.org/103"),
+                (301, "http://scrapytest.org/104"),
             ),
-            b'http://scrapytest.org/101',  # expected initial referer
-            b'http://scrapytest.org/101',  # expected referer for the redirection request
+            b"http://scrapytest.org/101",  # expected initial referer
+            b"http://scrapytest.org/101",  # expected referer for the redirection request
         ),
         (
-            'https://scrapytest.org/201',
-            'https://scrapytest.org/202',
+            "https://scrapytest.org/201",
+            "https://scrapytest.org/202",
             (
                 # redirecting to non-secure URL: do not send the "Referer" header
-                (301, 'http://scrapytest.org/203'),
+                (301, "http://scrapytest.org/203"),
             ),
-            b'https://scrapytest.org/201',
+            b"https://scrapytest.org/201",
             None,
         ),
         (
-            'https://scrapytest.org/301',
-            'https://scrapytest.org/302',
+            "https://scrapytest.org/301",
+            "https://scrapytest.org/302",
             (
                 # redirecting to non-secure URL (different domain): send origin
-                (301, 'http://example.com/303'),
+                (301, "http://example.com/303"),
             ),
-            b'https://scrapytest.org/301',
+            b"https://scrapytest.org/301",
             None,
         ),
         (
-            'http://scrapy.org/401',
-            'http://example.com/402',
-            (
-                (301, 'http://scrapytest.org/403'),
-            ),
-            b'http://scrapy.org/',
-            b'http://scrapy.org/',
+            "http://scrapy.org/401",
+            "http://example.com/402",
+            ((301, "http://scrapytest.org/403"),),
+            b"http://scrapy.org/",
+            b"http://scrapy.org/",
         ),
         (
-            'https://scrapy.org/501',
-            'https://example.com/502',
+            "https://scrapy.org/501",
+            "https://example.com/502",
             (
                 # all different domains: send origin
-                (301, 'https://google.com/503'),
-                (301, 'https://facebook.com/504'),
+                (301, "https://google.com/503"),
+                (301, "https://facebook.com/504"),
             ),
-            b'https://scrapy.org/',
-            b'https://scrapy.org/',
+            b"https://scrapy.org/",
+            b"https://scrapy.org/",
         ),
         (
-            'https://scrapytest.org/601',
-            'http://scrapytest.org/602',                # TLS to non-TLS: do not send "Referer"
+            "https://scrapytest.org/601",
+            "http://scrapytest.org/602",  # TLS to non-TLS: do not send "Referer"
             (
-                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) send nothing
+                (
+                    301,
+                    "https://scrapytest.org/603",
+                ),  # TLS URL again: (still) send nothing
             ),
             None,
             None,
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 171f4ddfd..22716bdda 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -10,23 +10,24 @@ from scrapy.settings import Settings
 
 
 class TestUrlLengthMiddleware(TestCase):
-
     def setUp(self):
         self.maxlength = 25
-        settings = Settings({'URLLENGTH_LIMIT': self.maxlength})
+        settings = Settings({"URLLENGTH_LIMIT": self.maxlength})
 
         crawler = get_crawler(Spider)
-        self.spider = crawler._create_spider('foo')
+        self.spider = crawler._create_spider("foo")
         self.stats = crawler.stats
         self.mw = UrlLengthMiddleware.from_settings(settings)
 
-        self.response = Response('http://scrapytest.org')
-        self.short_url_req = Request('http://scrapytest.org/')
-        self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
+        self.response = Response("http://scrapytest.org")
+        self.short_url_req = Request("http://scrapytest.org/")
+        self.long_url_req = Request("http://scrapytest.org/this_is_a_long_url")
         self.reqs = [self.short_url_req, self.long_url_req]
 
     def process_spider_output(self):
-        return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
+        return list(
+            self.mw.process_spider_output(self.response, self.reqs, self.spider)
+        )
 
     def test_middleware_works(self):
         self.assertEqual(self.process_spider_output(), [self.short_url_req])
@@ -35,7 +36,9 @@ class TestUrlLengthMiddleware(TestCase):
         with LogCapture() as log:
             self.process_spider_output()
 
-        ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider)
+        ric = self.stats.get_value(
+            "urllength/request_ignored_count", spider=self.spider
+        )
         self.assertEqual(ric, 1)
 
-        self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log))
+        self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log))
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index ab215576e..5c6dccf11 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -10,24 +10,23 @@ from scrapy.utils.test import get_crawler
 
 
 class SpiderStateTest(unittest.TestCase):
-
     def test_store_load(self):
         jobdir = self.mktemp()
         Path(jobdir).mkdir()
         try:
-            spider = Spider(name='default')
+            spider = Spider(name="default")
             dt = datetime.now()
 
             ss = SpiderState(jobdir)
             ss.spider_opened(spider)
-            spider.state['one'] = 1
-            spider.state['dt'] = dt
+            spider.state["one"] = 1
+            spider.state["dt"] = dt
             ss.spider_closed(spider)
 
-            spider2 = Spider(name='default')
+            spider2 = Spider(name="default")
             ss2 = SpiderState(jobdir)
             ss2.spider_opened(spider2)
-            self.assertEqual(spider.state, {'one': 1, 'dt': dt})
+            self.assertEqual(spider.state, {"one": 1, "dt": dt})
             ss2.spider_closed(spider2)
         finally:
             shutil.rmtree(jobdir)
@@ -35,7 +34,7 @@ class SpiderStateTest(unittest.TestCase):
     def test_state_attribute(self):
         # state attribute must be present if jobdir is not set, to provide a
         # consistent interface
-        spider = Spider(name='default')
+        spider = Spider(name="default")
         ss = SpiderState()
         ss.spider_opened(spider)
         self.assertEqual(spider.state, {})
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index acc821b83..0e2441f90 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -31,20 +31,19 @@ def nonserializable_object_test(self):
     q = self.queue()
     self.assertRaises(ValueError, q.push, lambda x: x)
     # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
-    sel = Selector(text='<html><body><p>some text</p></body></html>')
+    sel = Selector(text="<html><body><p>some text</p></body></html>")
     self.assertRaises(ValueError, q.push, sel)
 
 
 class FifoDiskQueueTestMixin:
-
     def test_serialize(self):
         q = self.queue()
-        q.push('a')
+        q.push("a")
         q.push(123)
-        q.push({'a': 'dict'})
-        self.assertEqual(q.pop(), 'a')
+        q.push({"a": "dict"})
+        self.assertEqual(q.pop(), "a")
         self.assertEqual(q.pop(), 123)
-        self.assertEqual(q.pop(), {'a': 'dict'})
+        self.assertEqual(q.pop(), {"a": "dict"})
 
     test_nonserializable_object = nonserializable_object_test
 
@@ -81,7 +80,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     def test_serialize_item(self):
         q = self.queue()
-        i = TestItem(name='foo')
+        i = TestItem(name="foo")
         q.push(i)
         i2 = q.pop()
         assert isinstance(i2, TestItem)
@@ -94,17 +93,17 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
         loader2 = q.pop()
         assert isinstance(loader2, TestLoader)
         assert loader2.default_item_class is TestItem
-        self.assertEqual(loader2.name_out('x'), 'xx')
+        self.assertEqual(loader2.name_out("x"), "xx")
 
     def test_serialize_request_recursive(self):
         q = self.queue()
-        r = Request('http://www.example.com')
-        r.meta['request'] = r
+        r = Request("http://www.example.com")
+        r.meta["request"] = r
         q.push(r)
         r2 = q.pop()
         assert isinstance(r2, Request)
         self.assertEqual(r.url, r2.url)
-        assert r2.meta['request'] is r2
+        assert r2.meta["request"] is r2
 
     def test_non_pickable_object(self):
         q = self.queue()
@@ -115,7 +114,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
                 self.assertIsInstance(exc.__context__, pickle.PicklingError)
             else:
                 self.assertIsInstance(exc.__context__, AttributeError)
-        sel = Selector(text='<html><body><p>some text</p></body></html>')
+        sel = Selector(text="<html><body><p>some text</p></body></html>")
         try:
             q.push(sel)
         except ValueError as exc:
@@ -139,33 +138,30 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
 
 
 class LifoDiskQueueTestMixin:
-
     def test_serialize(self):
         q = self.queue()
-        q.push('a')
+        q.push("a")
         q.push(123)
-        q.push({'a': 'dict'})
-        self.assertEqual(q.pop(), {'a': 'dict'})
+        q.push({"a": "dict"})
+        self.assertEqual(q.pop(), {"a": "dict"})
         self.assertEqual(q.pop(), 123)
-        self.assertEqual(q.pop(), 'a')
+        self.assertEqual(q.pop(), "a")
 
     test_nonserializable_object = nonserializable_object_test
 
 
 class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
-
     def queue(self):
         return _MarshalLifoSerializationDiskQueue(self.qpath)
 
 
 class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
-
     def queue(self):
         return _PickleLifoSerializationDiskQueue(self.qpath)
 
     def test_serialize_item(self):
         q = self.queue()
-        i = TestItem(name='foo')
+        i = TestItem(name="foo")
         q.push(i)
         i2 = q.pop()
         assert isinstance(i2, TestItem)
@@ -178,14 +174,14 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
         loader2 = q.pop()
         assert isinstance(loader2, TestLoader)
         assert loader2.default_item_class is TestItem
-        self.assertEqual(loader2.name_out('x'), 'xx')
+        self.assertEqual(loader2.name_out("x"), "xx")
 
     def test_serialize_request_recursive(self):
         q = self.queue()
-        r = Request('http://www.example.com')
-        r.meta['request'] = r
+        r = Request("http://www.example.com")
+        r.meta["request"] = r
         q.push(r)
         r2 = q.pop()
         assert isinstance(r2, Request)
         self.assertEqual(r.url, r2.url)
-        assert r2.meta['request'] is r2
+        assert r2.meta["request"] is r2
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index 22dafb2d2..5d9001bb0 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -69,7 +69,10 @@ class RequestQueueTestMixin:
         req = Request("http://www.example.com")
         q.push(req)
         self.assertEqual(len(q), 1)
-        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+        with self.assertRaises(
+            NotImplementedError,
+            msg="The underlying queue class does not implement 'peek'",
+        ):
             q.peek()
         self.assertEqual(q.pop().url, req.url)
         self.assertEqual(len(q), 0)
@@ -117,7 +120,10 @@ class FifoQueueMixin(RequestQueueTestMixin):
         q.push(req1)
         q.push(req2)
         q.push(req3)
-        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+        with self.assertRaises(
+            NotImplementedError,
+            msg="The underlying queue class does not implement 'peek'",
+        ):
             q.peek()
         self.assertEqual(len(q), 3)
         self.assertEqual(q.pop().url, req1.url)
@@ -170,7 +176,10 @@ class LifoQueueMixin(RequestQueueTestMixin):
         q.push(req1)
         q.push(req2)
         q.push(req3)
-        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+        with self.assertRaises(
+            NotImplementedError,
+            msg="The underlying queue class does not implement 'peek'",
+        ):
             q.peek()
         self.assertEqual(len(q), 3)
         self.assertEqual(q.pop().url, req3.url)
@@ -195,12 +204,16 @@ class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
 
 class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
     def queue(self):
-        return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo")
+        return MarshalFifoDiskQueue.from_crawler(
+            crawler=self.crawler, key="marshal/fifo"
+        )
 
 
 class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
     def queue(self):
-        return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo")
+        return MarshalLifoDiskQueue.from_crawler(
+            crawler=self.crawler, key="marshal/lifo"
+        )
 
 
 class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2bbbb9e2c..2ee04429a 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -9,12 +9,11 @@ from scrapy.utils.test import get_crawler
 
 
 class CoreStatsExtensionTest(unittest.TestCase):
-
     def setUp(self):
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('foo')
+        self.spider = self.crawler._create_spider("foo")
 
-    @mock.patch('scrapy.extensions.corestats.datetime')
+    @mock.patch("scrapy.extensions.corestats.datetime")
     def test_core_stats_default_stats_collector(self, mock_datetime):
         fixed_datetime = datetime(2019, 12, 1, 11, 38)
         mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
@@ -24,19 +23,19 @@ class CoreStatsExtensionTest(unittest.TestCase):
         ext.item_scraped({}, self.spider)
         ext.response_received(self.spider)
         ext.item_dropped({}, self.spider, ZeroDivisionError())
-        ext.spider_closed(self.spider, 'finished')
+        ext.spider_closed(self.spider, "finished")
         self.assertEqual(
             ext.stats._stats,
             {
-                'start_time': fixed_datetime,
-                'finish_time': fixed_datetime,
-                'item_scraped_count': 1,
-                'response_received_count': 1,
-                'item_dropped_count': 1,
-                'item_dropped_reasons_count/ZeroDivisionError': 1,
-                'finish_reason': 'finished',
-                'elapsed_time_seconds': 0.0,
-            }
+                "start_time": fixed_datetime,
+                "finish_time": fixed_datetime,
+                "item_scraped_count": 1,
+                "response_received_count": 1,
+                "item_dropped_count": 1,
+                "item_dropped_reasons_count/ZeroDivisionError": 1,
+                "finish_reason": "finished",
+                "elapsed_time_seconds": 0.0,
+            },
         )
 
     def test_core_stats_dummy_stats_collector(self):
@@ -46,53 +45,52 @@ class CoreStatsExtensionTest(unittest.TestCase):
         ext.item_scraped({}, self.spider)
         ext.response_received(self.spider)
         ext.item_dropped({}, self.spider, ZeroDivisionError())
-        ext.spider_closed(self.spider, 'finished')
+        ext.spider_closed(self.spider, "finished")
         self.assertEqual(ext.stats._stats, {})
 
 
 class StatsCollectorTest(unittest.TestCase):
-
     def setUp(self):
         self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('foo')
+        self.spider = self.crawler._create_spider("foo")
 
     def test_collector(self):
         stats = StatsCollector(self.crawler)
         self.assertEqual(stats.get_stats(), {})
-        self.assertEqual(stats.get_value('anything'), None)
-        self.assertEqual(stats.get_value('anything', 'default'), 'default')
-        stats.set_value('test', 'value')
-        self.assertEqual(stats.get_stats(), {'test': 'value'})
-        stats.set_value('test2', 23)
-        self.assertEqual(stats.get_stats(), {'test': 'value', 'test2': 23})
-        self.assertEqual(stats.get_value('test2'), 23)
-        stats.inc_value('test2')
-        self.assertEqual(stats.get_value('test2'), 24)
-        stats.inc_value('test2', 6)
-        self.assertEqual(stats.get_value('test2'), 30)
-        stats.max_value('test2', 6)
-        self.assertEqual(stats.get_value('test2'), 30)
-        stats.max_value('test2', 40)
-        self.assertEqual(stats.get_value('test2'), 40)
-        stats.max_value('test3', 1)
-        self.assertEqual(stats.get_value('test3'), 1)
-        stats.min_value('test2', 60)
-        self.assertEqual(stats.get_value('test2'), 40)
-        stats.min_value('test2', 35)
-        self.assertEqual(stats.get_value('test2'), 35)
-        stats.min_value('test4', 7)
-        self.assertEqual(stats.get_value('test4'), 7)
+        self.assertEqual(stats.get_value("anything"), None)
+        self.assertEqual(stats.get_value("anything", "default"), "default")
+        stats.set_value("test", "value")
+        self.assertEqual(stats.get_stats(), {"test": "value"})
+        stats.set_value("test2", 23)
+        self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23})
+        self.assertEqual(stats.get_value("test2"), 23)
+        stats.inc_value("test2")
+        self.assertEqual(stats.get_value("test2"), 24)
+        stats.inc_value("test2", 6)
+        self.assertEqual(stats.get_value("test2"), 30)
+        stats.max_value("test2", 6)
+        self.assertEqual(stats.get_value("test2"), 30)
+        stats.max_value("test2", 40)
+        self.assertEqual(stats.get_value("test2"), 40)
+        stats.max_value("test3", 1)
+        self.assertEqual(stats.get_value("test3"), 1)
+        stats.min_value("test2", 60)
+        self.assertEqual(stats.get_value("test2"), 40)
+        stats.min_value("test2", 35)
+        self.assertEqual(stats.get_value("test2"), 35)
+        stats.min_value("test4", 7)
+        self.assertEqual(stats.get_value("test4"), 7)
 
     def test_dummy_collector(self):
         stats = DummyStatsCollector(self.crawler)
         self.assertEqual(stats.get_stats(), {})
-        self.assertEqual(stats.get_value('anything'), None)
-        self.assertEqual(stats.get_value('anything', 'default'), 'default')
-        stats.set_value('test', 'value')
-        stats.inc_value('v1')
-        stats.max_value('v2', 100)
-        stats.min_value('v3', 100)
-        stats.open_spider('a')
-        stats.set_value('test', 'value', spider=self.spider)
+        self.assertEqual(stats.get_value("anything"), None)
+        self.assertEqual(stats.get_value("anything", "default"), "default")
+        stats.set_value("test", "value")
+        stats.inc_value("v1")
+        stats.max_value("v2", 100)
+        stats.min_value("v3", 100)
+        stats.open_spider("a")
+        stats.set_value("test", "value", spider=self.spider)
         self.assertEqual(stats.get_stats(), {})
-        self.assertEqual(stats.get_stats('a'), {})
+        self.assertEqual(stats.get_stats("a"), {})
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index fdc5df166..9a4eeb04e 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -4,7 +4,6 @@ import scrapy
 
 
 class ToplevelTestCase(TestCase):
-
     def test_version(self):
         self.assertIs(type(scrapy.__version__), str)
 
@@ -13,18 +12,22 @@ class ToplevelTestCase(TestCase):
 
     def test_request_shortcut(self):
         from scrapy.http import Request, FormRequest
+
         self.assertIs(scrapy.Request, Request)
         self.assertIs(scrapy.FormRequest, FormRequest)
 
     def test_spider_shortcut(self):
         from scrapy.spiders import Spider
+
         self.assertIs(scrapy.Spider, Spider)
 
     def test_selector_shortcut(self):
         from scrapy.selector import Selector
+
         self.assertIs(scrapy.Selector, Selector)
 
     def test_item_shortcut(self):
         from scrapy.item import Item, Field
+
         self.assertIs(scrapy.Item, Item)
         self.assertIs(scrapy.Field, Field)
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index bea0cf3e5..3b6428686 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -3,10 +3,9 @@ import unittest
 
 
 class UrlparseTestCase(unittest.TestCase):
-
     def test_s3_url(self):
-        p = urlparse('s3://bucket/key/name?param=value')
-        self.assertEqual(p.scheme, 's3')
-        self.assertEqual(p.hostname, 'bucket')
-        self.assertEqual(p.path, '/key/name')
-        self.assertEqual(p.query, 'param=value')
+        p = urlparse("s3://bucket/key/name?param=value")
+        self.assertEqual(p.scheme, "s3")
+        self.assertEqual(p.hostname, "bucket")
+        self.assertEqual(p.path, "/key/name")
+        self.assertEqual(p.query, "param=value")
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 741c6a505..25cef2b26 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -6,12 +6,13 @@ from pytest import mark
 from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor
 
 
-@mark.usefixtures('reactor_pytest')
+@mark.usefixtures("reactor_pytest")
 class AsyncioTest(TestCase):
-
     def test_is_asyncio_reactor_installed(self):
         # the result should depend only on the pytest --reactor argument
-        self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
+        self.assertEqual(
+            is_asyncio_reactor_installed(), self.reactor_pytest == "asyncio"
+        )
 
     def test_install_asyncio_reactor(self):
         with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index a92880626..61a683318 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -7,200 +7,242 @@ from scrapy.utils.conf import (
     arglist_to_dict,
     build_component_list,
     feed_complete_default_values_from_settings,
-    feed_process_params_from_cli
+    feed_process_params_from_cli,
 )
 
 
 class BuildComponentListTest(unittest.TestCase):
-
     def test_build_dict(self):
-        d = {'one': 1, 'two': None, 'three': 8, 'four': 4}
-        self.assertEqual(build_component_list(d, convert=lambda x: x),
-                         ['one', 'four', 'three'])
+        d = {"one": 1, "two": None, "three": 8, "four": 4}
+        self.assertEqual(
+            build_component_list(d, convert=lambda x: x), ["one", "four", "three"]
+        )
 
     def test_backward_compatible_build_dict(self):
-        base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
-        custom = {'two': None, 'three': 8, 'four': 4}
-        self.assertEqual(build_component_list(base, custom,
-                                              convert=lambda x: x),
-                         ['one', 'four', 'five', 'three'])
+        base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}
+        custom = {"two": None, "three": 8, "four": 4}
+        self.assertEqual(
+            build_component_list(base, custom, convert=lambda x: x),
+            ["one", "four", "five", "three"],
+        )
 
     def test_return_list(self):
-        custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(None, custom,
-                                              convert=lambda x: x),
-                         custom)
+        custom = ["a", "b", "c"]
+        self.assertEqual(
+            build_component_list(None, custom, convert=lambda x: x), custom
+        )
 
     def test_map_dict(self):
-        custom = {'one': 1, 'two': 2, 'three': 3}
-        self.assertEqual(build_component_list({}, custom,
-                                              convert=lambda x: x.upper()),
-                         ['ONE', 'TWO', 'THREE'])
+        custom = {"one": 1, "two": 2, "three": 3}
+        self.assertEqual(
+            build_component_list({}, custom, convert=lambda x: x.upper()),
+            ["ONE", "TWO", "THREE"],
+        )
 
     def test_map_list(self):
-        custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(None, custom,
-                                              lambda x: x.upper()),
-                         ['A', 'B', 'C'])
+        custom = ["a", "b", "c"]
+        self.assertEqual(
+            build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]
+        )
 
     def test_duplicate_components_in_dict(self):
-        duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4}
-        self.assertRaises(ValueError, build_component_list, {}, duplicate_dict,
-                          convert=lambda x: x.lower())
+        duplicate_dict = {"one": 1, "two": 2, "ONE": 4}
+        self.assertRaises(
+            ValueError,
+            build_component_list,
+            {},
+            duplicate_dict,
+            convert=lambda x: x.lower(),
+        )
 
     def test_duplicate_components_in_list(self):
-        duplicate_list = ['a', 'b', 'a']
+        duplicate_list = ["a", "b", "a"]
         with self.assertRaises(ValueError) as cm:
             build_component_list(None, duplicate_list, convert=lambda x: x)
         self.assertIn(str(duplicate_list), str(cm.exception))
 
     def test_duplicate_components_in_basesettings(self):
         # Higher priority takes precedence
-        duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
-        duplicate_bs.set('ONE', 4, priority=10)
-        self.assertEqual(build_component_list(duplicate_bs,
-                                              convert=lambda x: x.lower()),
-                         ['two', 'one'])
-        duplicate_bs.set('one', duplicate_bs['one'], priority=20)
-        self.assertEqual(build_component_list(duplicate_bs,
-                                              convert=lambda x: x.lower()),
-                         ['one', 'two'])
+        duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)
+        duplicate_bs.set("ONE", 4, priority=10)
+        self.assertEqual(
+            build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+            ["two", "one"],
+        )
+        duplicate_bs.set("one", duplicate_bs["one"], priority=20)
+        self.assertEqual(
+            build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+            ["one", "two"],
+        )
         # Same priority raises ValueError
-        duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
-        self.assertRaises(ValueError, build_component_list, duplicate_bs,
-                          convert=lambda x: x.lower())
+        duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
+        self.assertRaises(
+            ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower()
+        )
 
     def test_valid_numbers(self):
         # work well with None and numeric values
-        d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0}
-        self.assertEqual(build_component_list(d, convert=lambda x: x),
-                         ['d', 'a', 'c'])
-        d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222}
-        self.assertEqual(build_component_list(d, convert=lambda x: x),
-                         ['b', 'c', 'a'])
+        d = {"a": 10, "b": None, "c": 15, "d": 5.0}
+        self.assertEqual(build_component_list(d, convert=lambda x: x), ["d", "a", "c"])
+        d = {
+            "a": 33333333333333333333,
+            "b": 11111111111111111111,
+            "c": 22222222222222222222,
+        }
+        self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])
         # raise exception for invalid values
-        d = {'one': '5'}
+        d = {"one": "5"}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
-        d = {'one': '1.0'}
+        d = {"one": "1.0"}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
-        d = {'one': [1, 2, 3]}
+        d = {"one": [1, 2, 3]}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
-        d = {'one': {'a': 'a', 'b': 2}}
+        d = {"one": {"a": "a", "b": 2}}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
-        d = {'one': 'lorem ipsum'}
+        d = {"one": "lorem ipsum"}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
 
 
 class UtilsConfTestCase(unittest.TestCase):
-
     def test_arglist_to_dict(self):
         self.assertEqual(
-            arglist_to_dict(['arg1=val1', 'arg2=val2']),
-            {'arg1': 'val1', 'arg2': 'val2'})
+            arglist_to_dict(["arg1=val1", "arg2=val2"]),
+            {"arg1": "val1", "arg2": "val2"},
+        )
 
 
 class FeedExportConfigTestCase(unittest.TestCase):
-
     def test_feed_export_config_invalid_format(self):
         settings = Settings()
-        self.assertRaises(UsageError, feed_process_params_from_cli, settings, ['items.dat'], 'noformat')
+        self.assertRaises(
+            UsageError,
+            feed_process_params_from_cli,
+            settings,
+            ["items.dat"],
+            "noformat",
+        )
 
     def test_feed_export_config_mismatch(self):
         settings = Settings()
         self.assertRaises(
             UsageError,
-            feed_process_params_from_cli, settings, ['items1.dat', 'items2.dat'], 'noformat'
+            feed_process_params_from_cli,
+            settings,
+            ["items1.dat", "items2.dat"],
+            "noformat",
         )
 
     def test_feed_export_config_backward_compatible(self):
         with warnings.catch_warnings(record=True) as cw:
             settings = Settings()
             self.assertEqual(
-                {'items.dat': {'format': 'csv'}},
-                feed_process_params_from_cli(settings, ['items.dat'], 'csv')
+                {"items.dat": {"format": "csv"}},
+                feed_process_params_from_cli(settings, ["items.dat"], "csv"),
             )
             self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_feed_export_config_explicit_formats(self):
         settings = Settings()
         self.assertEqual(
-            {'items_1.dat': {'format': 'json'}, 'items_2.dat': {'format': 'xml'}, 'items_3.dat': {'format': 'csv'}},
-            feed_process_params_from_cli(settings, ['items_1.dat:json', 'items_2.dat:xml', 'items_3.dat:csv'])
+            {
+                "items_1.dat": {"format": "json"},
+                "items_2.dat": {"format": "xml"},
+                "items_3.dat": {"format": "csv"},
+            },
+            feed_process_params_from_cli(
+                settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"]
+            ),
         )
 
     def test_feed_export_config_implicit_formats(self):
         settings = Settings()
         self.assertEqual(
-            {'items_1.json': {'format': 'json'}, 'items_2.xml': {'format': 'xml'}, 'items_3.csv': {'format': 'csv'}},
-            feed_process_params_from_cli(settings, ['items_1.json', 'items_2.xml', 'items_3.csv'])
+            {
+                "items_1.json": {"format": "json"},
+                "items_2.xml": {"format": "xml"},
+                "items_3.csv": {"format": "csv"},
+            },
+            feed_process_params_from_cli(
+                settings, ["items_1.json", "items_2.xml", "items_3.csv"]
+            ),
         )
 
     def test_feed_export_config_stdout(self):
         settings = Settings()
         self.assertEqual(
-            {'stdout:': {'format': 'pickle'}},
-            feed_process_params_from_cli(settings, ['-:pickle'])
+            {"stdout:": {"format": "pickle"}},
+            feed_process_params_from_cli(settings, ["-:pickle"]),
         )
 
     def test_feed_export_config_overwrite(self):
         settings = Settings()
         self.assertEqual(
-            {'output.json': {'format': 'json', 'overwrite': True}},
-            feed_process_params_from_cli(settings, [], None, ['output.json'])
+            {"output.json": {"format": "json", "overwrite": True}},
+            feed_process_params_from_cli(settings, [], None, ["output.json"]),
         )
 
     def test_output_and_overwrite_output(self):
         with self.assertRaises(UsageError):
             feed_process_params_from_cli(
                 Settings(),
-                ['output1.json'],
+                ["output1.json"],
                 None,
-                ['output2.json'],
+                ["output2.json"],
             )
 
     def test_feed_complete_default_values_from_settings_empty(self):
         feed = {}
-        settings = Settings({
-            "FEED_EXPORT_ENCODING": "custom encoding",
-            "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
-            "FEED_EXPORT_INDENT": 42,
-            "FEED_STORE_EMPTY": True,
-            "FEED_URI_PARAMS": (1, 2, 3, 4),
-            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
-        })
+        settings = Settings(
+            {
+                "FEED_EXPORT_ENCODING": "custom encoding",
+                "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
+                "FEED_EXPORT_INDENT": 42,
+                "FEED_STORE_EMPTY": True,
+                "FEED_URI_PARAMS": (1, 2, 3, 4),
+                "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+            }
+        )
         new_feed = feed_complete_default_values_from_settings(feed, settings)
-        self.assertEqual(new_feed, {
-            "encoding": "custom encoding",
-            "fields": ["f1", "f2", "f3"],
-            "indent": 42,
-            "store_empty": True,
-            "uri_params": (1, 2, 3, 4),
-            "batch_item_count": 2,
-            "item_export_kwargs": {},
-        })
+        self.assertEqual(
+            new_feed,
+            {
+                "encoding": "custom encoding",
+                "fields": ["f1", "f2", "f3"],
+                "indent": 42,
+                "store_empty": True,
+                "uri_params": (1, 2, 3, 4),
+                "batch_item_count": 2,
+                "item_export_kwargs": {},
+            },
+        )
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
         feed = {
             "encoding": "other encoding",
             "fields": None,
         }
-        settings = Settings({
-            "FEED_EXPORT_ENCODING": "custom encoding",
-            "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
-            "FEED_EXPORT_INDENT": 42,
-            "FEED_STORE_EMPTY": True,
-            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
-        })
+        settings = Settings(
+            {
+                "FEED_EXPORT_ENCODING": "custom encoding",
+                "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
+                "FEED_EXPORT_INDENT": 42,
+                "FEED_STORE_EMPTY": True,
+                "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+            }
+        )
         new_feed = feed_complete_default_values_from_settings(feed, settings)
-        self.assertEqual(new_feed, {
-            "encoding": "other encoding",
-            "fields": None,
-            "indent": 42,
-            "store_empty": True,
-            "uri_params": None,
-            "batch_item_count": 2,
-            "item_export_kwargs": {},
-        })
+        self.assertEqual(
+            new_feed,
+            {
+                "encoding": "other encoding",
+                "fields": None,
+                "indent": 42,
+                "store_empty": True,
+                "uri_params": None,
+                "batch_item_count": 2,
+                "item_export_kwargs": {},
+            },
+        )
 
 
 if __name__ == "__main__":
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 380c41367..18f63b540 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -1,14 +1,17 @@
 import unittest
 
 from scrapy.utils.console import get_shell_embed_func
+
 try:
     import bpython
+
     bpy = True
     del bpython
 except ImportError:
     bpy = False
 try:
     import IPython
+
     ipy = True
     del IPython
 except ImportError:
@@ -16,29 +19,28 @@ except ImportError:
 
 
 class UtilsConsoleTestCase(unittest.TestCase):
-
     def test_get_shell_embed_func(self):
 
-        shell = get_shell_embed_func(['invalid'])
+        shell = get_shell_embed_func(["invalid"])
         self.assertEqual(shell, None)
 
-        shell = get_shell_embed_func(['invalid', 'python'])
+        shell = get_shell_embed_func(["invalid", "python"])
         self.assertTrue(callable(shell))
-        self.assertEqual(shell.__name__, '_embed_standard_shell')
+        self.assertEqual(shell.__name__, "_embed_standard_shell")
 
-    @unittest.skipIf(not bpy, 'bpython not available in testenv')
+    @unittest.skipIf(not bpy, "bpython not available in testenv")
     def test_get_shell_embed_func2(self):
 
-        shell = get_shell_embed_func(['bpython'])
+        shell = get_shell_embed_func(["bpython"])
         self.assertTrue(callable(shell))
-        self.assertEqual(shell.__name__, '_embed_bpython_shell')
+        self.assertEqual(shell.__name__, "_embed_bpython_shell")
 
-    @unittest.skipIf(not ipy, 'IPython not available in testenv')
+    @unittest.skipIf(not ipy, "IPython not available in testenv")
     def test_get_shell_embed_func3(self):
 
         # default shell should be 'ipython'
         shell = get_shell_embed_func()
-        self.assertEqual(shell.__name__, '_embed_ipython_shell')
+        self.assertEqual(shell.__name__, "_embed_ipython_shell")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index f5d684d3f..fd4612eba 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -34,10 +34,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
             "method": "GET",
             "url": "https://api.test.com/",
             "headers": [
-                (
-                    "Authorization",
-                    basic_auth_header("some_username", "some_password")
-                )
+                ("Authorization", basic_auth_header("some_username", "some_password"))
             ],
         }
         self._test_command(curl_command, expected_result)
@@ -77,11 +74,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):
                 ("Connection", "keep-alive"),
             ],
             "cookies": {
-                '_gauges_unique_year': '1',
-                '_gauges_unique_hour': '1',
-                '_gauges_unique_day': '1',
-                '_gauges_unique': '1',
-                '_gauges_unique_month': '1'
+                "_gauges_unique_year": "1",
+                "_gauges_unique_hour": "1",
+                "_gauges_unique_day": "1",
+                "_gauges_unique": "1",
+                "_gauges_unique_month": "1",
             },
         }
         self._test_command(curl_command, expected_result)
@@ -107,14 +104,14 @@ class CurlToRequestKwargsTest(unittest.TestCase):
             "method": "POST",
             "url": "http://httpbin.org/post",
             "body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl"
-                    "e.org&size=small&topping=cheese&topping=onion&delivery=12"
-                    "%3A15&comments=",
+            "e.org&size=small&topping=cheese&topping=onion&delivery=12"
+            "%3A15&comments=",
             "cookies": {
-                '_gauges_unique_year': '1',
-                '_gauges_unique_hour': '1',
-                '_gauges_unique_day': '1',
-                '_gauges_unique': '1',
-                '_gauges_unique_month': '1'
+                "_gauges_unique_year": "1",
+                "_gauges_unique_hour": "1",
+                "_gauges_unique_day": "1",
+                "_gauges_unique": "1",
+                "_gauges_unique_month": "1",
             },
             "headers": [
                 ("Origin", "http://httpbin.org"),
@@ -152,16 +149,17 @@ class CurlToRequestKwargsTest(unittest.TestCase):
             "body": (
                 "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
                 "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
-                "Functions=%5B%5D")
+                "Functions=%5B%5D"
+            ),
         }
         self._test_command(curl_command, expected_result)
 
     def test_explicit_get_with_data(self):
-        curl_command = 'curl httpbin.org/anything -X GET --data asdf'
+        curl_command = "curl httpbin.org/anything -X GET --data asdf"
         expected_result = {
             "method": "GET",
             "url": "http://httpbin.org/anything",
-            "body": "asdf"
+            "body": "asdf",
         }
         self._test_command(curl_command, expected_result)
 
@@ -182,16 +180,14 @@ class CurlToRequestKwargsTest(unittest.TestCase):
                 ("Authorization", basic_auth_header("username", "password")),
             ],
             "body": '{"hostname": "agent02.example.com",  "agent_config_state"'
-                    ': "Enabled", "resources": ["Java","Linux"], "environments'
-                    '": ["Dev"]}',
+            ': "Enabled", "resources": ["Java","Linux"], "environments'
+            '": ["Dev"]}',
         }
         self._test_command(curl_command, expected_result)
 
     def test_delete(self):
         curl_command = 'curl -X "DELETE" https://www.url.com/page'
-        expected_result = {
-            "method": "DELETE", "url": "https://www.url.com/page"
-        }
+        expected_result = {"method": "DELETE", "url": "https://www.url.com/page"}
         self._test_command(curl_command, expected_result)
 
     def test_get_silent(self):
@@ -209,8 +205,8 @@ class CurlToRequestKwargsTest(unittest.TestCase):
     def test_ignore_unknown_options(self):
         # case 1: ignore_unknown_options=True:
         with warnings.catch_warnings():  # avoid warning when executing tests
-            warnings.simplefilter('ignore')
-            curl_command = 'curl --bar --baz http://www.example.com'
+            warnings.simplefilter("ignore")
+            curl_command = "curl --bar --baz http://www.example.com"
             expected_result = {"method": "GET", "url": "http://www.example.com"}
             self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
@@ -219,13 +215,12 @@ class CurlToRequestKwargsTest(unittest.TestCase):
             ValueError,
             "Unrecognized options:.*--bar.*--baz",
             lambda: curl_to_request_kwargs(
-                "curl --bar --baz http://www.example.com",
-                ignore_unknown_options=False
+                "curl --bar --baz http://www.example.com", ignore_unknown_options=False
             ),
         )
 
     def test_must_start_with_curl_error(self):
         self.assertRaises(
             ValueError,
-            lambda: curl_to_request_kwargs("carl -X POST http://example.org")
+            lambda: curl_to_request_kwargs("carl -X POST http://example.org"),
         )
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index e4bccf30e..0c86c7e7a 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -3,26 +3,30 @@ import unittest
 from collections.abc import Mapping, MutableMapping
 
 from scrapy.http import Request
-from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude
+from scrapy.utils.datatypes import (
+    CaselessDict,
+    LocalCache,
+    LocalWeakReferencedCache,
+    SequenceExclude,
+)
 from scrapy.utils.python import garbage_collect
 
 
-__doctests__ = ['scrapy.utils.datatypes']
+__doctests__ = ["scrapy.utils.datatypes"]
 
 
 class CaselessDictTest(unittest.TestCase):
-
     def test_init_dict(self):
-        seq = {'red': 1, 'black': 3}
+        seq = {"red": 1, "black": 3}
         d = CaselessDict(seq)
-        self.assertEqual(d['red'], 1)
-        self.assertEqual(d['black'], 3)
+        self.assertEqual(d["red"], 1)
+        self.assertEqual(d["black"], 3)
 
     def test_init_pair_sequence(self):
-        seq = (('red', 1), ('black', 3))
+        seq = (("red", 1), ("black", 3))
         d = CaselessDict(seq)
-        self.assertEqual(d['red'], 1)
-        self.assertEqual(d['black'], 3)
+        self.assertEqual(d["red"], 1)
+        self.assertEqual(d["black"], 3)
 
     def test_init_mapping(self):
         class MyMapping(Mapping):
@@ -40,8 +44,8 @@ class CaselessDictTest(unittest.TestCase):
 
         seq = MyMapping(red=1, black=3)
         d = CaselessDict(seq)
-        self.assertEqual(d['red'], 1)
-        self.assertEqual(d['black'], 3)
+        self.assertEqual(d["red"], 1)
+        self.assertEqual(d["black"], 3)
 
     def test_init_mutable_mapping(self):
         class MyMutableMapping(MutableMapping):
@@ -65,72 +69,72 @@ class CaselessDictTest(unittest.TestCase):
 
         seq = MyMutableMapping(red=1, black=3)
         d = CaselessDict(seq)
-        self.assertEqual(d['red'], 1)
-        self.assertEqual(d['black'], 3)
+        self.assertEqual(d["red"], 1)
+        self.assertEqual(d["black"], 3)
 
     def test_caseless(self):
         d = CaselessDict()
-        d['key_Lower'] = 1
-        self.assertEqual(d['KEy_loWer'], 1)
-        self.assertEqual(d.get('KEy_loWer'), 1)
+        d["key_Lower"] = 1
+        self.assertEqual(d["KEy_loWer"], 1)
+        self.assertEqual(d.get("KEy_loWer"), 1)
 
-        d['KEY_LOWER'] = 3
-        self.assertEqual(d['key_Lower'], 3)
-        self.assertEqual(d.get('key_Lower'), 3)
+        d["KEY_LOWER"] = 3
+        self.assertEqual(d["key_Lower"], 3)
+        self.assertEqual(d.get("key_Lower"), 3)
 
     def test_delete(self):
-        d = CaselessDict({'key_lower': 1})
-        del d['key_LOWER']
-        self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
-        self.assertRaises(KeyError, d.__getitem__, 'key_lower')
+        d = CaselessDict({"key_lower": 1})
+        del d["key_LOWER"]
+        self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
+        self.assertRaises(KeyError, d.__getitem__, "key_lower")
 
     def test_getdefault(self):
         d = CaselessDict()
-        self.assertEqual(d.get('c', 5), 5)
-        d['c'] = 10
-        self.assertEqual(d.get('c', 5), 10)
+        self.assertEqual(d.get("c", 5), 5)
+        d["c"] = 10
+        self.assertEqual(d.get("c", 5), 10)
 
     def test_setdefault(self):
-        d = CaselessDict({'a': 1, 'b': 2})
+        d = CaselessDict({"a": 1, "b": 2})
 
-        r = d.setdefault('A', 5)
+        r = d.setdefault("A", 5)
         self.assertEqual(r, 1)
-        self.assertEqual(d['A'], 1)
+        self.assertEqual(d["A"], 1)
 
-        r = d.setdefault('c', 5)
+        r = d.setdefault("c", 5)
         self.assertEqual(r, 5)
-        self.assertEqual(d['C'], 5)
+        self.assertEqual(d["C"], 5)
 
     def test_fromkeys(self):
-        keys = ('a', 'b')
+        keys = ("a", "b")
 
         d = CaselessDict.fromkeys(keys)
-        self.assertEqual(d['A'], None)
-        self.assertEqual(d['B'], None)
+        self.assertEqual(d["A"], None)
+        self.assertEqual(d["B"], None)
 
         d = CaselessDict.fromkeys(keys, 1)
-        self.assertEqual(d['A'], 1)
-        self.assertEqual(d['B'], 1)
+        self.assertEqual(d["A"], 1)
+        self.assertEqual(d["B"], 1)
 
         instance = CaselessDict()
         d = instance.fromkeys(keys)
-        self.assertEqual(d['A'], None)
-        self.assertEqual(d['B'], None)
+        self.assertEqual(d["A"], None)
+        self.assertEqual(d["B"], None)
 
         d = instance.fromkeys(keys, 1)
-        self.assertEqual(d['A'], 1)
-        self.assertEqual(d['B'], 1)
+        self.assertEqual(d["A"], 1)
+        self.assertEqual(d["B"], 1)
 
     def test_contains(self):
         d = CaselessDict()
-        d['a'] = 1
-        assert 'a' in d
+        d["a"] = 1
+        assert "a" in d
 
     def test_pop(self):
         d = CaselessDict()
-        d['a'] = 1
-        self.assertEqual(d.pop('A'), 1)
-        self.assertRaises(KeyError, d.pop, 'A')
+        d["a"] = 1
+        self.assertEqual(d.pop("A"), 1)
+        self.assertRaises(KeyError, d.pop, "A")
 
     def test_normkey(self):
         class MyDict(CaselessDict):
@@ -138,8 +142,8 @@ class CaselessDictTest(unittest.TestCase):
                 return key.title()
 
         d = MyDict()
-        d['key-one'] = 2
-        self.assertEqual(list(d.keys()), ['Key-One'])
+        d["key-one"] = 2
+        self.assertEqual(list(d.keys()), ["Key-One"])
 
     def test_normvalue(self):
         class MyDict(CaselessDict):
@@ -147,39 +151,38 @@ class CaselessDictTest(unittest.TestCase):
                 if value is not None:
                     return value + 1
 
-        d = MyDict({'key': 1})
-        self.assertEqual(d['key'], 2)
-        self.assertEqual(d.get('key'), 2)
+        d = MyDict({"key": 1})
+        self.assertEqual(d["key"], 2)
+        self.assertEqual(d.get("key"), 2)
 
         d = MyDict()
-        d['key'] = 1
-        self.assertEqual(d['key'], 2)
-        self.assertEqual(d.get('key'), 2)
+        d["key"] = 1
+        self.assertEqual(d["key"], 2)
+        self.assertEqual(d.get("key"), 2)
 
         d = MyDict()
-        d.setdefault('key', 1)
-        self.assertEqual(d['key'], 2)
-        self.assertEqual(d.get('key'), 2)
+        d.setdefault("key", 1)
+        self.assertEqual(d["key"], 2)
+        self.assertEqual(d.get("key"), 2)
 
         d = MyDict()
-        d.update({'key': 1})
-        self.assertEqual(d['key'], 2)
-        self.assertEqual(d.get('key'), 2)
+        d.update({"key": 1})
+        self.assertEqual(d["key"], 2)
+        self.assertEqual(d.get("key"), 2)
 
-        d = MyDict.fromkeys(('key',), 1)
-        self.assertEqual(d['key'], 2)
-        self.assertEqual(d.get('key'), 2)
+        d = MyDict.fromkeys(("key",), 1)
+        self.assertEqual(d["key"], 2)
+        self.assertEqual(d.get("key"), 2)
 
     def test_copy(self):
-        h1 = CaselessDict({'header1': 'value'})
+        h1 = CaselessDict({"header1": "value"})
         h2 = copy.copy(h1)
         self.assertEqual(h1, h2)
-        self.assertEqual(h1.get('header1'), h2.get('header1'))
+        self.assertEqual(h1.get("header1"), h2.get("header1"))
         assert isinstance(h2, CaselessDict)
 
 
 class SequenceExcludeTest(unittest.TestCase):
-
     def test_list(self):
         seq = [1, 2, 3]
         d = SequenceExclude(seq)
@@ -226,25 +229,24 @@ class SequenceExcludeTest(unittest.TestCase):
 
         # supplied sequence is a set, so checking for list (non)inclusion fails
         self.assertRaises(TypeError, (0, 1, 2) in d)
-        self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c'])
+        self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"])
 
         for v in [-3, "test", 1.1]:
             self.assertNotIn(v, d)
 
 
 class LocalCacheTest(unittest.TestCase):
-
     def test_cache_with_limit(self):
         cache = LocalCache(limit=2)
-        cache['a'] = 1
-        cache['b'] = 2
-        cache['c'] = 3
+        cache["a"] = 1
+        cache["b"] = 2
+        cache["c"] = 3
         self.assertEqual(len(cache), 2)
-        self.assertNotIn('a', cache)
-        self.assertIn('b', cache)
-        self.assertIn('c', cache)
-        self.assertEqual(cache['b'], 2)
-        self.assertEqual(cache['c'], 3)
+        self.assertNotIn("a", cache)
+        self.assertIn("b", cache)
+        self.assertIn("c", cache)
+        self.assertEqual(cache["b"], 2)
+        self.assertEqual(cache["c"], 3)
 
     def test_cache_without_limit(self):
         maximum = 10**4
@@ -258,12 +260,11 @@ class LocalCacheTest(unittest.TestCase):
 
 
 class LocalWeakReferencedCacheTest(unittest.TestCase):
-
     def test_cache_with_limit(self):
         cache = LocalWeakReferencedCache(limit=2)
-        r1 = Request('https://example.org')
-        r2 = Request('https://example.com')
-        r3 = Request('https://example.net')
+        r1 = Request("https://example.org")
+        r2 = Request("https://example.com")
+        r3 = Request("https://example.net")
         cache[r1] = 1
         cache[r2] = 2
         cache[r3] = 3
@@ -299,7 +300,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
         cache = LocalWeakReferencedCache()
         refs = []
         for x in range(max):
-            refs.append(Request(f'https://example.org/{x}'))
+            refs.append(Request(f"https://example.org/{x}"))
             cache[refs[-1]] = x
         self.assertEqual(len(cache), max)
         for i, r in enumerate(refs):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 97c1c60d1..8d7f33c9a 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -68,41 +68,43 @@ def eb1(failure, arg1, arg2):
 
 
 class DeferUtilsTest(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_process_chain(self):
-        x = yield process_chain([cb1, cb2, cb3], 'res', 'v1', 'v2')
+        x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2")
         self.assertEqual(x, "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)")
 
         gotexc = False
         try:
-            yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
+            yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")
         except TypeError:
             gotexc = True
         self.assertTrue(gotexc)
 
     @defer.inlineCallbacks
     def test_process_chain_both(self):
-        x = yield process_chain_both([cb_fail, cb2, cb3], [None, eb1, None], 'res', 'v1', 'v2')
+        x = yield process_chain_both(
+            [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2"
+        )
         self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)")
 
         fail = Failure(ZeroDivisionError())
-        x = yield process_chain_both([eb1, cb2, cb3], [eb1, None, None], fail, 'v1', 'v2')
+        x = yield process_chain_both(
+            [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2"
+        )
         self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)")
 
     @defer.inlineCallbacks
     def test_process_parallel(self):
-        x = yield process_parallel([cb1, cb2, cb3], 'res', 'v1', 'v2')
-        self.assertEqual(x, ['(cb1 res v1 v2)', '(cb2 res v1 v2)', '(cb3 res v1 v2)'])
+        x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
+        self.assertEqual(x, ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"])
 
     def test_process_parallel_failure(self):
-        d = process_parallel([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
+        d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
         self.failUnlessFailure(d, TypeError)
         return d
 
 
 class IterErrbackTest(unittest.TestCase):
-
     def test_iter_errback_good(self):
         def itergood():
             for x in range(10):
@@ -128,7 +130,6 @@ class IterErrbackTest(unittest.TestCase):
 
 
 class AiterErrbackTest(unittest.TestCase):
-
     @deferred_f_from_coro_f
     async def test_aiter_errback_good(self):
         async def itergood():
@@ -171,7 +172,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
 
 
 class AsyncCooperatorTest(unittest.TestCase):
-    """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
+    """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
 
     parallel_async is called with the results of a callback (so an iterable of items, requests and None,
     with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable
@@ -182,6 +183,7 @@ class AsyncCooperatorTest(unittest.TestCase):
     We also want to simulate the real usage, with arbitrary delays between getting the values
     from the iterable. We also want to simulate sync and async results from the callable.
     """
+
     CONCURRENT_ITEMS = 50
 
     @staticmethod
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 50c63dfab..1a8c96e7a 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -20,22 +20,23 @@ class NewName(SomeBaseClass):
 
 
 class WarnWhenSubclassedTest(unittest.TestCase):
-
     def _mywarnings(self, w, category=MyWarning):
         return [x for x in w if x.category is MyWarning]
 
     def test_no_warning_on_definition(self):
         with warnings.catch_warnings(record=True) as w:
-            create_deprecated_class('Deprecated', NewName)
+            create_deprecated_class("Deprecated", NewName)
 
         w = self._mywarnings(w)
         self.assertEqual(w, [])
 
     def test_subclassing_warning_message(self):
-        Deprecated = create_deprecated_class('Deprecated', NewName,
-                                             warn_category=MyWarning)
+        Deprecated = create_deprecated_class(
+            "Deprecated", NewName, warn_category=MyWarning
+        )
 
         with warnings.catch_warnings(record=True) as w:
+
             class UserClass(Deprecated):
                 pass
 
@@ -46,17 +47,21 @@ class WarnWhenSubclassedTest(unittest.TestCase):
             "tests.test_utils_deprecate.UserClass inherits from "
             "deprecated class tests.test_utils_deprecate.Deprecated, "
             "please inherit from tests.test_utils_deprecate.NewName."
-            " (warning only on first subclass, there may be others)"
+            " (warning only on first subclass, there may be others)",
         )
         self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
 
     def test_custom_class_paths(self):
-        Deprecated = create_deprecated_class('Deprecated', NewName,
-                                             new_class_path='foo.NewClass',
-                                             old_class_path='bar.OldClass',
-                                             warn_category=MyWarning)
+        Deprecated = create_deprecated_class(
+            "Deprecated",
+            NewName,
+            new_class_path="foo.NewClass",
+            old_class_path="bar.OldClass",
+            warn_category=MyWarning,
+        )
 
         with warnings.catch_warnings(record=True) as w:
+
             class UserClass(Deprecated):
                 pass
 
@@ -64,17 +69,18 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
         w = self._mywarnings(w)
         self.assertEqual(len(w), 2)
-        self.assertIn('foo.NewClass', str(w[0].message))
-        self.assertIn('bar.OldClass', str(w[0].message))
-        self.assertIn('foo.NewClass', str(w[1].message))
-        self.assertIn('bar.OldClass', str(w[1].message))
+        self.assertIn("foo.NewClass", str(w[0].message))
+        self.assertIn("bar.OldClass", str(w[0].message))
+        self.assertIn("foo.NewClass", str(w[1].message))
+        self.assertIn("bar.OldClass", str(w[1].message))
 
     def test_subclassing_warns_only_on_direct_childs(self):
-        Deprecated = create_deprecated_class('Deprecated', NewName,
-                                             warn_once=False,
-                                             warn_category=MyWarning)
+        Deprecated = create_deprecated_class(
+            "Deprecated", NewName, warn_once=False, warn_category=MyWarning
+        )
 
         with warnings.catch_warnings(record=True) as w:
+
             class UserClass(Deprecated):
                 pass
 
@@ -83,13 +89,15 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
         w = self._mywarnings(w)
         self.assertEqual(len(w), 1)
-        self.assertIn('UserClass', str(w[0].message))
+        self.assertIn("UserClass", str(w[0].message))
 
     def test_subclassing_warns_once_by_default(self):
-        Deprecated = create_deprecated_class('Deprecated', NewName,
-                                             warn_category=MyWarning)
+        Deprecated = create_deprecated_class(
+            "Deprecated", NewName, warn_category=MyWarning
+        )
 
         with warnings.catch_warnings(record=True) as w:
+
             class UserClass(Deprecated):
                 pass
 
@@ -101,15 +109,16 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
         w = self._mywarnings(w)
         self.assertEqual(len(w), 1)
-        self.assertIn('UserClass', str(w[0].message))
+        self.assertIn("UserClass", str(w[0].message))
 
     def test_warning_on_instance(self):
-        Deprecated = create_deprecated_class('Deprecated', NewName,
-                                             warn_category=MyWarning)
+        Deprecated = create_deprecated_class(
+            "Deprecated", NewName, warn_category=MyWarning
+        )
 
         # ignore subclassing warnings
         with warnings.catch_warnings():
-            warnings.simplefilter('ignore', MyWarning)
+            warnings.simplefilter("ignore", MyWarning)
 
             class UserClass(Deprecated):
                 pass
@@ -123,13 +132,13 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         self.assertEqual(
             str(w[0].message),
             "tests.test_utils_deprecate.Deprecated is deprecated, "
-            "instantiate tests.test_utils_deprecate.NewName instead."
+            "instantiate tests.test_utils_deprecate.NewName instead.",
         )
         self.assertEqual(w[0].lineno, lineno)
 
     def test_warning_auto_message(self):
         with warnings.catch_warnings(record=True) as w:
-            Deprecated = create_deprecated_class('Deprecated', NewName)
+            Deprecated = create_deprecated_class("Deprecated", NewName)
 
             class UserClass2(Deprecated):
                 pass
@@ -140,8 +149,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
     def test_issubclass(self):
         with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
 
             class UpdatedUserClass1(NewName):
                 pass
@@ -176,8 +185,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
     def test_isinstance(self):
         with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
 
             class UpdatedUserClass2(NewName):
                 pass
@@ -210,24 +219,28 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
     def test_clsdict(self):
         with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'})
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
 
-        self.assertEqual(Deprecated.foo, 'bar')
+        self.assertEqual(Deprecated.foo, "bar")
 
     def test_deprecate_a_class_with_custom_metaclass(self):
-        Meta1 = type('Meta1', (type,), {})
-        New = Meta1('New', (), {})
-        create_deprecated_class('Deprecated', New)
+        Meta1 = type("Meta1", (type,), {})
+        New = Meta1("New", (), {})
+        create_deprecated_class("Deprecated", New)
 
     def test_deprecate_subclass_of_deprecated_class(self):
         with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            Deprecated = create_deprecated_class('Deprecated', NewName,
-                                                 warn_category=MyWarning)
-            AlsoDeprecated = create_deprecated_class('AlsoDeprecated', Deprecated,
-                                                     new_class_path='foo.Bar',
-                                                     warn_category=MyWarning)
+            warnings.simplefilter("always")
+            Deprecated = create_deprecated_class(
+                "Deprecated", NewName, warn_category=MyWarning
+            )
+            AlsoDeprecated = create_deprecated_class(
+                "AlsoDeprecated",
+                Deprecated,
+                new_class_path="foo.Bar",
+                warn_category=MyWarning,
+            )
 
         w = self._mywarnings(w)
         self.assertEqual(len(w), 0, str(map(str, w)))
@@ -240,15 +253,15 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
         w = self._mywarnings(w)
         self.assertEqual(len(w), 2)
-        self.assertIn('AlsoDeprecated', str(w[0].message))
-        self.assertIn('foo.Bar', str(w[0].message))
-        self.assertIn('AlsoDeprecated', str(w[1].message))
-        self.assertIn('foo.Bar', str(w[1].message))
+        self.assertIn("AlsoDeprecated", str(w[0].message))
+        self.assertIn("foo.Bar", str(w[0].message))
+        self.assertIn("AlsoDeprecated", str(w[1].message))
+        self.assertIn("foo.Bar", str(w[1].message))
 
     def test_inspect_stack(self):
-        with mock.patch('inspect.stack', side_effect=IndexError):
+        with mock.patch("inspect.stack", side_effect=IndexError):
             with warnings.catch_warnings(record=True) as w:
-                DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+                DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
 
                 class SubClass(DeprecatedName):
                     pass
@@ -256,27 +269,30 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         self.assertIn("Error detecting parent module", str(w[0].message))
 
 
-@mock.patch('scrapy.utils.deprecate.DEPRECATION_RULES',
-            [('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
-             ('scrapy.contrib.', 'scrapy.extensions.')])
+@mock.patch(
+    "scrapy.utils.deprecate.DEPRECATION_RULES",
+    [
+        ("scrapy.contrib.pipeline.", "scrapy.pipelines."),
+        ("scrapy.contrib.", "scrapy.extensions."),
+    ],
+)
 class UpdateClassPathTest(unittest.TestCase):
-
     def test_old_path_gets_fixed(self):
         with warnings.catch_warnings(record=True) as w:
-            output = update_classpath('scrapy.contrib.debug.Debug')
-        self.assertEqual(output, 'scrapy.extensions.debug.Debug')
+            output = update_classpath("scrapy.contrib.debug.Debug")
+        self.assertEqual(output, "scrapy.extensions.debug.Debug")
         self.assertEqual(len(w), 1)
         self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message))
         self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message))
 
     def test_sorted_replacement(self):
         with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            output = update_classpath('scrapy.contrib.pipeline.Pipeline')
-        self.assertEqual(output, 'scrapy.pipelines.Pipeline')
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            output = update_classpath("scrapy.contrib.pipeline.Pipeline")
+        self.assertEqual(output, "scrapy.pipelines.Pipeline")
 
     def test_unmatched_path_stays_the_same(self):
         with warnings.catch_warnings(record=True) as w:
-            output = update_classpath('scrapy.unmatched.Path')
-        self.assertEqual(output, 'scrapy.unmatched.Path')
+            output = update_classpath("scrapy.unmatched.Path")
+        self.assertEqual(output, "scrapy.unmatched.Path")
         self.assertEqual(len(w), 0)
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 9ec8311d9..43236acec 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -6,14 +6,14 @@ from scrapy.utils.display import pformat, pprint
 
 
 class TestDisplay(TestCase):
-    object = {'a': 1}
+    object = {"a": 1}
     colorized_string = (
         "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
         "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
     )
     plain_string = "{'a': 1}"
 
-    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.platform", "linux")
     @mock.patch("sys.stdout.isatty")
     def test_pformat(self, isatty):
         isatty.return_value = True
@@ -27,44 +27,47 @@ class TestDisplay(TestCase):
     def test_pformat_not_tty(self):
         self.assertEqual(pformat(self.object), self.plain_string)
 
-    @mock.patch('sys.platform', 'win32')
-    @mock.patch('platform.version')
+    @mock.patch("sys.platform", "win32")
+    @mock.patch("platform.version")
     @mock.patch("sys.stdout.isatty")
     def test_pformat_old_windows(self, isatty, version):
         isatty.return_value = True
-        version.return_value = '10.0.14392'
+        version.return_value = "10.0.14392"
         self.assertEqual(pformat(self.object), self.colorized_string)
 
-    @mock.patch('sys.platform', 'win32')
-    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
-    @mock.patch('platform.version')
+    @mock.patch("sys.platform", "win32")
+    @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
+    @mock.patch("platform.version")
     @mock.patch("sys.stdout.isatty")
-    def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing):
+    def test_pformat_windows_no_terminal_processing(
+        self, isatty, version, terminal_processing
+    ):
         isatty.return_value = True
-        version.return_value = '10.0.14393'
+        version.return_value = "10.0.14393"
         terminal_processing.return_value = False
         self.assertEqual(pformat(self.object), self.plain_string)
 
-    @mock.patch('sys.platform', 'win32')
-    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
-    @mock.patch('platform.version')
+    @mock.patch("sys.platform", "win32")
+    @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
+    @mock.patch("platform.version")
     @mock.patch("sys.stdout.isatty")
     def test_pformat_windows(self, isatty, version, terminal_processing):
         isatty.return_value = True
-        version.return_value = '10.0.14393'
+        version.return_value = "10.0.14393"
         terminal_processing.return_value = True
         self.assertEqual(pformat(self.object), self.colorized_string)
 
-    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.platform", "linux")
     @mock.patch("sys.stdout.isatty")
     def test_pformat_no_pygments(self, isatty):
         isatty.return_value = True
 
         import builtins
+
         real_import = builtins.__import__
 
         def mock_import(name, globals, locals, fromlist, level):
-            if 'pygments' in name:
+            if "pygments" in name:
                 raise ImportError
             return real_import(name, globals, locals, fromlist, level)
 
@@ -73,6 +76,6 @@ class TestDisplay(TestCase):
         builtins.__import__ = real_import
 
     def test_pprint(self):
-        with mock.patch('sys.stdout', new=StringIO()) as mock_out:
+        with mock.patch("sys.stdout", new=StringIO()) as mock_out:
             pprint(self.object)
             self.assertEqual(mock_out.getvalue(), "{'a': 1}\n")
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index ca98bff21..a34664956 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -8,13 +8,15 @@ from scrapy.http import Response
 from tests import tests_datadir
 
 
-SAMPLEDIR = Path(tests_datadir, 'compressed')
+SAMPLEDIR = Path(tests_datadir, "compressed")
 
 
 class GunzipTest(unittest.TestCase):
-
     def test_gunzip_basic(self):
-        r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes())
+        r1 = Response(
+            "http://www.example.com",
+            body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
+        )
         self.assertTrue(gzip_magic_number(r1))
 
         r2 = Response("http://www.example.com", body=gunzip(r1.body))
@@ -22,18 +24,23 @@ class GunzipTest(unittest.TestCase):
         self.assertEqual(len(r2.body), 9950)
 
     def test_gunzip_truncated(self):
-        text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes())
-        assert text.endswith(b'</html')
+        text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
+        assert text.endswith(b"</html")
 
     def test_gunzip_no_gzip_file_raises(self):
-        self.assertRaises(IOError, gunzip, (SAMPLEDIR / 'feed-sample1.xml').read_bytes())
+        self.assertRaises(
+            IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
+        )
 
     def test_gunzip_truncated_short(self):
-        r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'truncated-crc-error-short.gz').read_bytes())
+        r1 = Response(
+            "http://www.example.com",
+            body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
+        )
         self.assertTrue(gzip_magic_number(r1))
 
         r2 = Response("http://www.example.com", body=gunzip(r1.body))
-        assert r2.body.endswith(b'</html>')
+        assert r2.body.endswith(b"</html>")
         self.assertFalse(gzip_magic_number(r2))
 
     def test_is_gzipped_empty(self):
@@ -41,7 +48,11 @@ class GunzipTest(unittest.TestCase):
         self.assertFalse(gzip_magic_number(r1))
 
     def test_gunzip_illegal_eof(self):
-        text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1]
-        expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8")
+        text = html_to_unicode(
+            "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
+        )[1]
+        expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(
+            encoding="utf-8"
+        )
         self.assertEqual(len(text), len(expected_text))
         self.assertEqual(text, expected_text)
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index cf8ad1f23..b824972d5 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -6,7 +6,6 @@ from scrapy.utils.httpobj import urlparse_cached
 
 
 class HttpobjUtilsTest(unittest.TestCase):
-
     def test_urlparse_cached(self):
         url = "http://www.example.com/index.html"
         request1 = Request(url)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ba3136b96..893582a32 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -28,14 +28,18 @@ class XmliterTestCase(unittest.TestCase):
 
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
-        for x in self.xmliter(response, 'product'):
-            attrs.append((
-                x.attrib['id'],
-                x.xpath("name/text()").getall(),
-                x.xpath("./type/text()").getall()))
+        for x in self.xmliter(response, "product"):
+            attrs.append(
+                (
+                    x.attrib["id"],
+                    x.xpath("name/text()").getall(),
+                    x.xpath("./type/text()").getall(),
+                )
+            )
 
-        self.assertEqual(attrs,
-                         [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
+        self.assertEqual(
+            attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])]
+        )
 
     def test_xmliter_unusual_node(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -45,8 +49,10 @@ class XmliterTestCase(unittest.TestCase):
             </root>
         """
         response = XmlResponse(url="http://example.com", body=body)
-        nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
-        self.assertEqual(nodenames, [['matchme...']])
+        nodenames = [
+            e.xpath("name()").getall() for e in self.xmliter(response, "matchme...")
+        ]
+        self.assertEqual(nodenames, [["matchme..."]])
 
     def test_xmliter_unicode(self):
         # example taken from https://github.com/scrapy/scrapy/issues/1665
@@ -88,29 +94,35 @@ class XmliterTestCase(unittest.TestCase):
 
         for r in (
             # with bytes
-            XmlResponse(url="http://example.com", body=body.encode('utf-8')),
+            XmlResponse(url="http://example.com", body=body.encode("utf-8")),
             # Unicode body needs encoding information
-            XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
+            XmlResponse(url="http://example.com", body=body, encoding="utf-8"),
         ):
             attrs = []
-            for x in self.xmliter(r, 'þingflokkur'):
-                attrs.append((x.attrib['id'],
-                              x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
-                              x.xpath('./tímabil/fyrstaþing/text()').getall()))
+            for x in self.xmliter(r, "þingflokkur"):
+                attrs.append(
+                    (
+                        x.attrib["id"],
+                        x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
+                        x.xpath("./tímabil/fyrstaþing/text()").getall(),
+                    )
+                )
 
-            self.assertEqual(attrs,
-                             [('26', ['-'], ['80']),
-                              ('21', ['Ab'], ['76']),
-                              ('27', ['A'], ['27'])])
+            self.assertEqual(
+                attrs,
+                [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])],
+            )
 
     def test_xmliter_text(self):
         body = (
             '<?xml version="1.0" encoding="UTF-8"?>'
-            '<products><product>one</product><product>two</product></products>'
+            "<products><product>one</product><product>two</product></products>"
         )
 
-        self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
-                         [['one'], ['two']])
+        self.assertEqual(
+            [x.xpath("text()").getall() for x in self.xmliter(body, "product")],
+            [["one"], ["two"]],
+        )
 
     def test_xmliter_namespaces(self):
         body = b"""
@@ -131,22 +143,25 @@ class XmliterTestCase(unittest.TestCase):
                 </channel>
             </rss>
         """
-        response = XmlResponse(url='http://mydummycompany.com', body=body)
-        my_iter = self.xmliter(response, 'item')
+        response = XmlResponse(url="http://mydummycompany.com", body=body)
+        my_iter = self.xmliter(response, "item")
         node = next(my_iter)
-        node.register_namespace('g', 'http://base.google.com/ns/1.0')
-        self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
-        self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
-        self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+        node.register_namespace("g", "http://base.google.com/ns/1.0")
+        self.assertEqual(node.xpath("title/text()").getall(), ["Item 1"])
+        self.assertEqual(node.xpath("description/text()").getall(), ["This is item 1"])
         self.assertEqual(
-            node.xpath('g:image_link/text()').getall(),
-            ['http://www.mydummycompany.com/images/item1.jpg']
+            node.xpath("link/text()").getall(),
+            ["http://www.mydummycompany.com/items/1"],
         )
-        self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
-        self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
-        self.assertEqual(node.xpath('image_link/text()').getall(), [])
-        self.assertEqual(node.xpath('id/text()').getall(), [])
-        self.assertEqual(node.xpath('price/text()').getall(), [])
+        self.assertEqual(
+            node.xpath("g:image_link/text()").getall(),
+            ["http://www.mydummycompany.com/images/item1.jpg"],
+        )
+        self.assertEqual(node.xpath("g:id/text()").getall(), ["ITEM_1"])
+        self.assertEqual(node.xpath("g:price/text()").getall(), ["400"])
+        self.assertEqual(node.xpath("image_link/text()").getall(), [])
+        self.assertEqual(node.xpath("id/text()").getall(), [])
+        self.assertEqual(node.xpath("price/text()").getall(), [])
 
     def test_xmliter_namespaced_nodename(self):
         body = b"""
@@ -167,11 +182,14 @@ class XmliterTestCase(unittest.TestCase):
                 </channel>
             </rss>
         """
-        response = XmlResponse(url='http://mydummycompany.com', body=body)
-        my_iter = self.xmliter(response, 'g:image_link')
+        response = XmlResponse(url="http://mydummycompany.com", body=body)
+        my_iter = self.xmliter(response, "g:image_link")
         node = next(my_iter)
-        node.register_namespace('g', 'http://base.google.com/ns/1.0')
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        node.register_namespace("g", "http://base.google.com/ns/1.0")
+        self.assertEqual(
+            node.xpath("text()").extract(),
+            ["http://www.mydummycompany.com/images/item1.jpg"],
+        )
 
     def test_xmliter_namespaced_nodename_missing(self):
         body = b"""
@@ -192,45 +210,45 @@ class XmliterTestCase(unittest.TestCase):
                 </channel>
             </rss>
         """
-        response = XmlResponse(url='http://mydummycompany.com', body=body)
-        my_iter = self.xmliter(response, 'g:link_image')
+        response = XmlResponse(url="http://mydummycompany.com", body=body)
+        my_iter = self.xmliter(response, "g:link_image")
         with self.assertRaises(StopIteration):
             next(my_iter)
 
     def test_xmliter_exception(self):
         body = (
             '<?xml version="1.0" encoding="UTF-8"?>'
-            '<products><product>one</product><product>two</product></products>'
+            "<products><product>one</product><product>two</product></products>"
         )
 
-        iter = self.xmliter(body, 'product')
+        iter = self.xmliter(body, "product")
         next(iter)
         next(iter)
 
         self.assertRaises(StopIteration, next, iter)
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, 'product')
+        i = self.xmliter(42, "product")
         self.assertRaises(TypeError, next, i)
 
     def test_xmliter_encoding(self):
         body = (
             b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
-            b'<xml>\n'
-            b'    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n'
-            b'</xml>\n\n'
+            b"<xml>\n"
+            b"    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n"
+            b"</xml>\n\n"
         )
-        response = XmlResponse('http://www.example.com', body=body)
+        response = XmlResponse("http://www.example.com", body=body)
         self.assertEqual(
-            next(self.xmliter(response, 'item')).get(),
-            '<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
+            next(self.xmliter(response, "item")).get(),
+            "<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>",
         )
 
 
 class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
-    @mark.xfail(reason='known bug of the current implementation')
+    @mark.xfail(reason="known bug of the current implementation")
     def test_xmliter_namespaced_nodename(self):
         super().test_xmliter_namespaced_nodename()
 
@@ -252,16 +270,24 @@ class LxmlXmliterTestCase(XmliterTestCase):
                 </channel>
             </rss>
         """
-        response = XmlResponse(url='http://mydummycompany.com', body=body)
+        response = XmlResponse(url="http://mydummycompany.com", body=body)
 
-        no_namespace_iter = self.xmliter(response, 'image_link')
+        no_namespace_iter = self.xmliter(response, "image_link")
         self.assertEqual(len(list(no_namespace_iter)), 0)
 
-        namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
+        namespace_iter = self.xmliter(
+            response, "image_link", "http://base.google.com/ns/1.0"
+        )
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(
+            node.xpath("text()").getall(),
+            ["http://www.mydummycompany.com/images/item1.jpg"],
+        )
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
+        self.assertEqual(
+            node.xpath("text()").getall(),
+            ["http://www.mydummycompany.com/images/item2.jpg"],
+        )
 
     def test_xmliter_namespaces_prefix(self):
         body = b"""
@@ -282,36 +308,42 @@ class LxmlXmliterTestCase(XmliterTestCase):
 
         </root>
         """
-        response = XmlResponse(url='http://mydummycompany.com', body=body)
-        my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
+        response = XmlResponse(url="http://mydummycompany.com", body=body)
+        my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h")
 
         node = next(my_iter)
-        self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
-        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
-        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
+        self.assertEqual(len(node.xpath("h:tr/h:td").getall()), 2)
+        self.assertEqual(node.xpath("h:tr/h:td[1]/text()").getall(), ["Apples"])
+        self.assertEqual(node.xpath("h:tr/h:td[2]/text()").getall(), ["Bananas"])
 
-        my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
+        my_iter = self.xmliter(
+            response, "table", "http://www.w3schools.com/furniture", "f"
+        )
 
         node = next(my_iter)
-        self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
+        self.assertEqual(node.xpath("f:name/text()").getall(), ["African Coffee Table"])
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, 'product')
+        i = self.xmliter(42, "product")
         self.assertRaises(TypeError, next, i)
 
 
 class UtilsCsvTestCase(unittest.TestCase):
     def test_csviter_defaults(self):
-        body = get_testdata('feeds', 'feed-sample3.csv')
+        body = get_testdata("feeds", "feed-sample3.csv")
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response)
 
         result = [row for row in csv]
-        self.assertEqual(result,
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            result,
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
@@ -319,88 +351,120 @@ class UtilsCsvTestCase(unittest.TestCase):
             self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
-        body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
+        body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")
         response = TextResponse(url="http://example.com/", body=body)
-        csv = csviter(response, delimiter='\t')
+        csv = csviter(response, delimiter="\t")
 
-        self.assertEqual([row for row in csv],
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            [row for row in csv],
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
     def test_csviter_quotechar(self):
-        body1 = get_testdata('feeds', 'feed-sample6.csv')
-        body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|')
+        body1 = get_testdata("feeds", "feed-sample6.csv")
+        body2 = get_testdata("feeds", "feed-sample6.csv").replace(b",", b"|")
 
         response1 = TextResponse(url="http://example.com/", body=body1)
         csv1 = csviter(response1, quotechar="'")
 
-        self.assertEqual([row for row in csv1],
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            [row for row in csv1],
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
         response2 = TextResponse(url="http://example.com/", body=body2)
         csv2 = csviter(response2, delimiter="|", quotechar="'")
 
-        self.assertEqual([row for row in csv2],
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            [row for row in csv2],
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
     def test_csviter_wrong_quotechar(self):
-        body = get_testdata('feeds', 'feed-sample6.csv')
+        body = get_testdata("feeds", "feed-sample6.csv")
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response)
 
-        self.assertEqual([row for row in csv],
-                         [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
-                          {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
-                          {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
-                          {"'id'": "4", "'name'": "'empty'", "'value'": ""}])
+        self.assertEqual(
+            [row for row in csv],
+            [
+                {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
+                {
+                    "'id'": "2",
+                    "'name'": "'unicode'",
+                    "'value'": "'\xfan\xedc\xf3d\xe9\u203d'",
+                },
+                {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
+                {"'id'": "4", "'name'": "'empty'", "'value'": ""},
+            ],
+        )
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
-        body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
+        body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")
         response = Response(url="http://example.com/", body=body)
-        csv = csviter(response, delimiter='\t')
+        csv = csviter(response, delimiter="\t")
 
-        self.assertEqual([row for row in csv],
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            [row for row in csv],
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
     def test_csviter_headers(self):
-        sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
-        headers, body = sample[0].split(b','), b'\n'.join(sample[1:])
+        sample = get_testdata("feeds", "feed-sample3.csv").splitlines()
+        headers, body = sample[0].split(b","), b"\n".join(sample[1:])
 
         response = TextResponse(url="http://example.com/", body=body)
-        csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
+        csv = csviter(response, headers=[h.decode("utf-8") for h in headers])
 
-        self.assertEqual([row for row in csv],
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            [row for row in csv],
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
     def test_csviter_falserow(self):
-        body = get_testdata('feeds', 'feed-sample3.csv')
-        body = b'\n'.join((body, b'a,b', b'a,b,c,d'))
+        body = get_testdata("feeds", "feed-sample3.csv")
+        body = b"\n".join((body, b"a,b", b"a,b,c,d"))
 
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response)
 
-        self.assertEqual([row for row in csv],
-                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
-                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
-                          {'id': '4', 'name': 'empty', 'value': ''}])
+        self.assertEqual(
+            [row for row in csv],
+            [
+                {"id": "1", "name": "alpha", "value": "foobar"},
+                {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+                {"id": "3", "name": "multi", "value": "foo\nbar"},
+                {"id": "4", "name": "empty", "value": ""},
+            ],
+        )
 
     def test_csviter_exception(self):
-        body = get_testdata('feeds', 'feed-sample3.csv')
+        body = get_testdata("feeds", "feed-sample3.csv")
 
         response = TextResponse(url="http://example.com/", body=body)
         iter = csviter(response)
@@ -412,35 +476,41 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertRaises(StopIteration, next, iter)
 
     def test_csviter_encoding(self):
-        body1 = get_testdata('feeds', 'feed-sample4.csv')
-        body2 = get_testdata('feeds', 'feed-sample5.csv')
+        body1 = get_testdata("feeds", "feed-sample4.csv")
+        body2 = get_testdata("feeds", "feed-sample5.csv")
 
-        response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
+        response = TextResponse(
+            url="http://example.com/", body=body1, encoding="latin1"
+        )
         csv = csviter(response)
         self.assertEqual(
             list(csv),
             [
-                {'id': '1', 'name': 'latin1', 'value': 'test'},
-                {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
-            ]
+                {"id": "1", "name": "latin1", "value": "test"},
+                {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"},
+            ],
         )
 
-        response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
+        response = TextResponse(url="http://example.com/", body=body2, encoding="cp852")
         csv = csviter(response)
         self.assertEqual(
             list(csv),
             [
-                {'id': '1', 'name': 'cp852', 'value': 'test'},
-                {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
-            ]
+                {"id": "1", "name": "cp852", "value": "test"},
+                {
+                    "id": "2",
+                    "name": "something",
+                    "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557",
+                },
+            ],
         )
 
 
 class TestHelper(unittest.TestCase):
-    bbody = b'utf8-body'
-    ubody = bbody.decode('utf8')
-    txtresponse = TextResponse(url='http://example.org/', body=bbody, encoding='utf-8')
-    response = Response(url='http://example.org/', body=bbody)
+    bbody = b"utf8-body"
+    ubody = bbody.decode("utf8")
+    txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8")
+    response = Response(url="http://example.org/", body=bbody)
 
     def test_body_or_str(self):
         for obj in (self.bbody, self.ubody, self.txtresponse, self.response):
@@ -454,8 +524,9 @@ class TestHelper(unittest.TestCase):
             self.assertTrue(type(r1) is not type(r3))
 
     def _assert_type_and_value(self, a, b, obj):
-        self.assertTrue(type(a) is type(b),
-                        f'Got {type(a)}, expected {type(b)} for { obj!r}')
+        self.assertTrue(
+            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
+        )
         self.assertEqual(a, b)
 
 
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 535f56691..438dd0cdc 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -5,14 +5,17 @@ import unittest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
-from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter,
-                              LogCounterHandler, StreamLogger)
+from scrapy.utils.log import (
+    failure_to_exc_info,
+    TopLevelFormatter,
+    LogCounterHandler,
+    StreamLogger,
+)
 from scrapy.utils.test import get_crawler
 from scrapy.extensions import telnet
 
 
 class FailureToExcInfoTest(unittest.TestCase):
-
     def test_failure(self):
         try:
             0 / 0
@@ -23,48 +26,46 @@ class FailureToExcInfoTest(unittest.TestCase):
         self.assertTupleEqual(exc_info, failure_to_exc_info(failure))
 
     def test_non_failure(self):
-        self.assertIsNone(failure_to_exc_info('test'))
+        self.assertIsNone(failure_to_exc_info("test"))
 
 
 class TopLevelFormatterTest(unittest.TestCase):
-
     def setUp(self):
         self.handler = LogCapture()
-        self.handler.addFilter(TopLevelFormatter(['test']))
+        self.handler.addFilter(TopLevelFormatter(["test"]))
 
     def test_top_level_logger(self):
-        logger = logging.getLogger('test')
+        logger = logging.getLogger("test")
         with self.handler as log:
-            logger.warning('test log msg')
-        log.check(('test', 'WARNING', 'test log msg'))
+            logger.warning("test log msg")
+        log.check(("test", "WARNING", "test log msg"))
 
     def test_children_logger(self):
-        logger = logging.getLogger('test.test1')
+        logger = logging.getLogger("test.test1")
         with self.handler as log:
-            logger.warning('test log msg')
-        log.check(('test', 'WARNING', 'test log msg'))
+            logger.warning("test log msg")
+        log.check(("test", "WARNING", "test log msg"))
 
     def test_overlapping_name_logger(self):
-        logger = logging.getLogger('test2')
+        logger = logging.getLogger("test2")
         with self.handler as log:
-            logger.warning('test log msg')
-        log.check(('test2', 'WARNING', 'test log msg'))
+            logger.warning("test log msg")
+        log.check(("test2", "WARNING", "test log msg"))
 
     def test_different_name_logger(self):
-        logger = logging.getLogger('different')
+        logger = logging.getLogger("different")
         with self.handler as log:
-            logger.warning('test log msg')
-        log.check(('different', 'WARNING', 'test log msg'))
+            logger.warning("test log msg")
+        log.check(("different", "WARNING", "test log msg"))
 
 
 class LogCounterHandlerTest(unittest.TestCase):
-
     def setUp(self):
-        settings = {'LOG_LEVEL': 'WARNING'}
+        settings = {"LOG_LEVEL": "WARNING"}
         if not telnet.TWISTED_CONCH_AVAILABLE:
             # disable it to avoid the extra warning
-            settings['TELNETCONSOLE_ENABLED'] = False
-        self.logger = logging.getLogger('test')
+            settings["TELNETCONSOLE_ENABLED"] = False
+        self.logger = logging.getLogger("test")
         self.logger.setLevel(logging.NOTSET)
         self.logger.propagate = False
         self.crawler = get_crawler(settings_dict=settings)
@@ -76,26 +77,25 @@ class LogCounterHandlerTest(unittest.TestCase):
         self.logger.removeHandler(self.handler)
 
     def test_init(self):
-        self.assertIsNone(self.crawler.stats.get_value('log_count/DEBUG'))
-        self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))
-        self.assertIsNone(self.crawler.stats.get_value('log_count/WARNING'))
-        self.assertIsNone(self.crawler.stats.get_value('log_count/ERROR'))
-        self.assertIsNone(self.crawler.stats.get_value('log_count/CRITICAL'))
+        self.assertIsNone(self.crawler.stats.get_value("log_count/DEBUG"))
+        self.assertIsNone(self.crawler.stats.get_value("log_count/INFO"))
+        self.assertIsNone(self.crawler.stats.get_value("log_count/WARNING"))
+        self.assertIsNone(self.crawler.stats.get_value("log_count/ERROR"))
+        self.assertIsNone(self.crawler.stats.get_value("log_count/CRITICAL"))
 
     def test_accepted_level(self):
-        self.logger.error('test log msg')
-        self.assertEqual(self.crawler.stats.get_value('log_count/ERROR'), 1)
+        self.logger.error("test log msg")
+        self.assertEqual(self.crawler.stats.get_value("log_count/ERROR"), 1)
 
     def test_filtered_out_level(self):
-        self.logger.debug('test log msg')
-        self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))
+        self.logger.debug("test log msg")
+        self.assertIsNone(self.crawler.stats.get_value("log_count/INFO"))
 
 
 class StreamLoggerTest(unittest.TestCase):
-
     def setUp(self):
         self.stdout = sys.stdout
-        logger = logging.getLogger('test')
+        logger = logging.getLogger("test")
         logger.setLevel(logging.WARNING)
         sys.stdout = StreamLogger(logger, logging.ERROR)
 
@@ -104,5 +104,5 @@ class StreamLoggerTest(unittest.TestCase):
 
     def test_redirect(self):
         with LogCapture() as log:
-            print('test log msg')
-        log.check(('test', 'ERROR', 'test log msg'))
+            print("test log msg")
+        log.check(("test", "ERROR", "test log msg"))
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index dc5b9e123..38a61036c 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -5,118 +5,122 @@ from pathlib import Path
 from unittest import mock
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules
+from scrapy.utils.misc import (
+    arg_to_iter,
+    create_instance,
+    load_object,
+    rel_has_nofollow,
+    set_environ,
+    walk_modules,
+)
 
 
-__doctests__ = ['scrapy.utils.misc']
+__doctests__ = ["scrapy.utils.misc"]
 
 
 class UtilsMiscTestCase(unittest.TestCase):
-
     def test_load_object_class(self):
         obj = load_object(Field)
         self.assertIs(obj, Field)
-        obj = load_object('scrapy.item.Field')
+        obj = load_object("scrapy.item.Field")
         self.assertIs(obj, Field)
 
     def test_load_object_function(self):
         obj = load_object(load_object)
         self.assertIs(obj, load_object)
-        obj = load_object('scrapy.utils.misc.load_object')
+        obj = load_object("scrapy.utils.misc.load_object")
         self.assertIs(obj, load_object)
 
     def test_load_object_exceptions(self):
-        self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
-        self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
+        self.assertRaises(ImportError, load_object, "nomodule999.mod.function")
+        self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999")
         self.assertRaises(TypeError, load_object, {})
 
     def test_walk_modules(self):
-        mods = walk_modules('tests.test_utils_misc.test_walk_modules')
+        mods = walk_modules("tests.test_utils_misc.test_walk_modules")
         expected = [
-            'tests.test_utils_misc.test_walk_modules',
-            'tests.test_utils_misc.test_walk_modules.mod',
-            'tests.test_utils_misc.test_walk_modules.mod.mod0',
-            'tests.test_utils_misc.test_walk_modules.mod1',
+            "tests.test_utils_misc.test_walk_modules",
+            "tests.test_utils_misc.test_walk_modules.mod",
+            "tests.test_utils_misc.test_walk_modules.mod.mod0",
+            "tests.test_utils_misc.test_walk_modules.mod1",
         ]
         self.assertEqual({m.__name__ for m in mods}, set(expected))
 
-        mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod')
+        mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod")
         expected = [
-            'tests.test_utils_misc.test_walk_modules.mod',
-            'tests.test_utils_misc.test_walk_modules.mod.mod0',
+            "tests.test_utils_misc.test_walk_modules.mod",
+            "tests.test_utils_misc.test_walk_modules.mod.mod0",
         ]
         self.assertEqual({m.__name__ for m in mods}, set(expected))
 
-        mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1')
+        mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1")
         expected = [
-            'tests.test_utils_misc.test_walk_modules.mod1',
+            "tests.test_utils_misc.test_walk_modules.mod1",
         ]
         self.assertEqual({m.__name__ for m in mods}, set(expected))
 
-        self.assertRaises(ImportError, walk_modules, 'nomodule999')
+        self.assertRaises(ImportError, walk_modules, "nomodule999")
 
     def test_walk_modules_egg(self):
-        egg = str(Path(__file__).parent / 'test.egg')
+        egg = str(Path(__file__).parent / "test.egg")
         sys.path.append(egg)
         try:
-            mods = walk_modules('testegg')
+            mods = walk_modules("testegg")
             expected = [
-                'testegg.spiders',
-                'testegg.spiders.a',
-                'testegg.spiders.b',
-                'testegg'
+                "testegg.spiders",
+                "testegg.spiders.a",
+                "testegg.spiders.b",
+                "testegg",
             ]
             self.assertEqual({m.__name__ for m in mods}, set(expected))
         finally:
             sys.path.remove(egg)
 
     def test_arg_to_iter(self):
-
         class TestItem(Item):
             name = Field()
 
-        assert hasattr(arg_to_iter(None), '__iter__')
-        assert hasattr(arg_to_iter(100), '__iter__')
-        assert hasattr(arg_to_iter('lala'), '__iter__')
-        assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
-        assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__')
+        assert hasattr(arg_to_iter(None), "__iter__")
+        assert hasattr(arg_to_iter(100), "__iter__")
+        assert hasattr(arg_to_iter("lala"), "__iter__")
+        assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
+        assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
 
         self.assertEqual(list(arg_to_iter(None)), [])
-        self.assertEqual(list(arg_to_iter('lala')), ['lala'])
+        self.assertEqual(list(arg_to_iter("lala")), ["lala"])
         self.assertEqual(list(arg_to_iter(100)), [100])
-        self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c'])
+        self.assertEqual(list(arg_to_iter(c for c in "abc")), ["a", "b", "c"])
         self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
-        self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
-        self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
+        self.assertEqual(list(arg_to_iter({"a": 1})), [{"a": 1}])
+        self.assertEqual(
+            list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]
+        )
 
     def test_create_instance(self):
         settings = mock.MagicMock()
-        crawler = mock.MagicMock(spec_set=['settings'])
-        args = (True, 100.)
-        kwargs = {'key': 'val'}
+        crawler = mock.MagicMock(spec_set=["settings"])
+        args = (True, 100.0)
+        kwargs = {"key": "val"}
 
         def _test_with_settings(mock, settings):
             create_instance(mock, settings, None, *args, **kwargs)
-            if hasattr(mock, 'from_crawler'):
+            if hasattr(mock, "from_crawler"):
                 self.assertEqual(mock.from_crawler.call_count, 0)
-            if hasattr(mock, 'from_settings'):
-                mock.from_settings.assert_called_once_with(settings, *args,
-                                                           **kwargs)
+            if hasattr(mock, "from_settings"):
+                mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
                 self.assertEqual(mock.call_count, 0)
             else:
                 mock.assert_called_once_with(*args, **kwargs)
 
         def _test_with_crawler(mock, settings, crawler):
             create_instance(mock, settings, crawler, *args, **kwargs)
-            if hasattr(mock, 'from_crawler'):
-                mock.from_crawler.assert_called_once_with(crawler, *args,
-                                                          **kwargs)
-                if hasattr(mock, 'from_settings'):
+            if hasattr(mock, "from_crawler"):
+                mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
+                if hasattr(mock, "from_settings"):
                     self.assertEqual(mock.from_settings.call_count, 0)
                 self.assertEqual(mock.call_count, 0)
-            elif hasattr(mock, 'from_settings'):
-                mock.from_settings.assert_called_once_with(settings, *args,
-                                                           **kwargs)
+            elif hasattr(mock, "from_settings"):
+                mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
                 self.assertEqual(mock.call_count, 0)
             else:
                 mock.assert_called_once_with(*args, **kwargs)
@@ -127,10 +131,10 @@ class UtilsMiscTestCase(unittest.TestCase):
         #   3. with from_crawler() constructor
         #   4. with from_settings() and from_crawler() constructor
         spec_sets = (
-            ['__qualname__'],
-            ['__qualname__', 'from_settings'],
-            ['__qualname__', 'from_crawler'],
-            ['__qualname__', 'from_settings', 'from_crawler'],
+            ["__qualname__"],
+            ["__qualname__", "from_settings"],
+            ["__qualname__", "from_crawler"],
+            ["__qualname__", "from_settings", "from_crawler"],
         )
         for specs in spec_sets:
             m = mock.MagicMock(spec_set=specs)
@@ -139,10 +143,9 @@ class UtilsMiscTestCase(unittest.TestCase):
             _test_with_crawler(m, settings, crawler)
 
         # Check adoption of crawler settings
-        m = mock.MagicMock(spec_set=['__qualname__', 'from_settings'])
+        m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
         create_instance(m, None, crawler, *args, **kwargs)
-        m.from_settings.assert_called_once_with(crawler.settings, *args,
-                                                **kwargs)
+        m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
 
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
@@ -152,25 +155,25 @@ class UtilsMiscTestCase(unittest.TestCase):
             create_instance(m, settings, None)
 
     def test_set_environ(self):
-        assert os.environ.get('some_test_environ') is None
-        with set_environ(some_test_environ='test_value'):
-            assert os.environ.get('some_test_environ') == 'test_value'
-        assert os.environ.get('some_test_environ') is None
+        assert os.environ.get("some_test_environ") is None
+        with set_environ(some_test_environ="test_value"):
+            assert os.environ.get("some_test_environ") == "test_value"
+        assert os.environ.get("some_test_environ") is None
 
-        os.environ['some_test_environ'] = 'test'
-        assert os.environ.get('some_test_environ') == 'test'
-        with set_environ(some_test_environ='test_value'):
-            assert os.environ.get('some_test_environ') == 'test_value'
-        assert os.environ.get('some_test_environ') == 'test'
+        os.environ["some_test_environ"] = "test"
+        assert os.environ.get("some_test_environ") == "test"
+        with set_environ(some_test_environ="test_value"):
+            assert os.environ.get("some_test_environ") == "test_value"
+        assert os.environ.get("some_test_environ") == "test"
 
     def test_rel_has_nofollow(self):
-        assert rel_has_nofollow('ugc nofollow') is True
-        assert rel_has_nofollow('ugc,nofollow') is True
-        assert rel_has_nofollow('ugc') is False
-        assert rel_has_nofollow('nofollow') is True
-        assert rel_has_nofollow('nofollowfoo') is False
-        assert rel_has_nofollow('foonofollow') is False
-        assert rel_has_nofollow('ugc,  ,  nofollow') is True
+        assert rel_has_nofollow("ugc nofollow") is True
+        assert rel_has_nofollow("ugc,nofollow") is True
+        assert rel_has_nofollow("ugc") is False
+        assert rel_has_nofollow("nofollow") is True
+        assert rel_has_nofollow("nofollowfoo") is False
+        assert rel_has_nofollow("foonofollow") is False
+        assert rel_has_nofollow("ugc,  ,  nofollow") is True
 
 
 if __name__ == "__main__":
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 562f72fee..484757035 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -3,7 +3,10 @@ import warnings
 from functools import partial
 from unittest import mock
 
-from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value
+from scrapy.utils.misc import (
+    is_generator_with_return_value,
+    warn_on_generator_with_return_value,
+)
 
 
 def _indentation_error(*args, **kwargs):
@@ -12,7 +15,7 @@ def _indentation_error(*args, **kwargs):
 
 def top_level_return_something():
     """
-docstring
+    docstring
     """
     url = """
 https://example.org
@@ -23,7 +26,7 @@ https://example.org
 
 def top_level_return_none():
     """
-docstring
+    docstring
     """
     url = """
 https://example.org
@@ -39,7 +42,6 @@ def generator_that_returns_stuff():
 
 
 class UtilsMiscPy3TestCase(unittest.TestCase):
-
     def test_generators_return_something(self):
         def f1():
             yield 1
@@ -60,7 +62,7 @@ class UtilsMiscPy3TestCase(unittest.TestCase):
 
         def i1():
             """
-docstring
+            docstring
             """
             url = """
 https://example.org
@@ -77,7 +79,10 @@ https://example.org
         with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(None, top_level_return_something)
             self.assertEqual(len(w), 1)
-            self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message))
+            self.assertIn(
+                'The "NoneType.top_level_return_something" method is a generator',
+                str(w[0].message),
+            )
         with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(None, f1)
             self.assertEqual(len(w), 1)
@@ -121,7 +126,7 @@ https://example.org
 
         def k2():
             """
-docstring
+            docstring
             """
             url = """
 https://example.org
@@ -170,6 +175,7 @@ https://example.org
         def decorator(func):
             def inner_func():
                 func()
+
             return inner_func
 
         @decorator
@@ -203,7 +209,7 @@ https://example.org
         @decorator
         def k3():
             """
-docstring
+            docstring
             """
             url = """
 https://example.org
@@ -249,12 +255,14 @@ https://example.org
             warn_on_generator_with_return_value(None, l3)
             self.assertEqual(len(w), 0)
 
-    @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
+    @mock.patch(
+        "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
+    )
     def test_indentation_error(self):
         with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(None, top_level_return_none)
             self.assertEqual(len(w), 1)
-            self.assertIn('Unable to determine', str(w[0].message))
+            self.assertIn("Unable to determine", str(w[0].message))
 
     def test_partial(self):
         def cb(arg1, arg2):
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 867e12103..b08e5f475 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -16,7 +16,7 @@ def inside_a_project():
 
     try:
         os.chdir(project_dir)
-        Path('scrapy.cfg').touch()
+        Path("scrapy.cfg").touch()
 
         yield project_dir
     finally:
@@ -26,21 +26,15 @@ def inside_a_project():
 
 class ProjectUtilsTest(unittest.TestCase):
     def test_data_path_outside_project(self):
-        self.assertEqual(
-            str(Path('.scrapy', 'somepath')),
-            data_path('somepath')
-        )
-        abspath = str(Path(os.path.sep, 'absolute', 'path'))
+        self.assertEqual(str(Path(".scrapy", "somepath")), data_path("somepath"))
+        abspath = str(Path(os.path.sep, "absolute", "path"))
         self.assertEqual(abspath, data_path(abspath))
 
     def test_data_path_inside_project(self):
         with inside_a_project() as proj_path:
-            expected = Path(proj_path, '.scrapy', 'somepath')
-            self.assertEqual(
-                expected.resolve(),
-                Path(data_path('somepath')).resolve()
-            )
-            abspath = str(Path(os.path.sep, 'absolute', 'path').resolve())
+            expected = Path(proj_path, ".scrapy", "somepath")
+            self.assertEqual(expected.resolve(), Path(data_path("somepath")).resolve())
+            abspath = str(Path(os.path.sep, "absolute", "path").resolve())
             self.assertEqual(abspath, data_path(abspath))
 
 
@@ -59,22 +53,21 @@ def set_env(**update):
 
 
 class GetProjectSettingsTestCase(unittest.TestCase):
-
     def test_valid_envvar(self):
-        value = 'tests.test_cmdline.settings'
+        value = "tests.test_cmdline.settings"
         envvars = {
-            'SCRAPY_SETTINGS_MODULE': value,
+            "SCRAPY_SETTINGS_MODULE": value,
         }
         with warnings.catch_warnings():
             warnings.simplefilter("error")
             with set_env(**envvars):
                 settings = get_project_settings()
 
-        assert settings.get('SETTINGS_MODULE') == value
+        assert settings.get("SETTINGS_MODULE") == value
 
     def test_invalid_envvar(self):
         envvars = {
-            'SCRAPY_FOO': 'bar',
+            "SCRAPY_FOO": "bar",
         }
         with set_env(**envvars):
             settings = get_project_settings()
@@ -82,12 +75,12 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         assert settings.get("SCRAPY_FOO") is None
 
     def test_valid_and_invalid_envvars(self):
-        value = 'tests.test_cmdline.settings'
+        value = "tests.test_cmdline.settings"
         envvars = {
-            'SCRAPY_FOO': 'bar',
-            'SCRAPY_SETTINGS_MODULE': value,
+            "SCRAPY_FOO": "bar",
+            "SCRAPY_SETTINGS_MODULE": value,
         }
         with set_env(**envvars):
             settings = get_project_settings()
-        assert settings.get('SETTINGS_MODULE') == value
-        assert settings.get('SCRAPY_FOO') is None
+        assert settings.get("SETTINGS_MODULE") == value
+        assert settings.get("SCRAPY_FOO") is None
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 403e4f8fe..3c6270864 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -7,12 +7,19 @@ from twisted.trial import unittest
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback
 from scrapy.utils.python import (
-    memoizemethod_noargs, binary_is_text, equal_attributes,
-    get_func_args, to_bytes, to_unicode,
-    without_none_values, MutableChain, MutableAsyncChain)
+    memoizemethod_noargs,
+    binary_is_text,
+    equal_attributes,
+    get_func_args,
+    to_bytes,
+    to_unicode,
+    without_none_values,
+    MutableChain,
+    MutableAsyncChain,
+)
 
 
-__doctests__ = ['scrapy.utils.python']
+__doctests__ = ["scrapy.utils.python"]
 
 
 class MutableChainTest(unittest.TestCase):
@@ -79,48 +86,41 @@ class MutableAsyncChainTest(unittest.TestCase):
 
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
+        self.assertEqual(to_unicode(b"lel\xc3\xb1e"), "lel\xf1e")
 
     def test_converting_a_latin_1_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
+        self.assertEqual(to_unicode(b"lel\xf1e", "latin-1"), "lel\xf1e")
 
     def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
-        self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
+        self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e")
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_unicode, 423)
 
     def test_errors_argument(self):
-        self.assertEqual(
-            to_unicode(b'a\xedb', 'utf-8', errors='replace'),
-            'a\ufffdb'
-        )
+        self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb")
 
 
 class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
-        self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
+        self.assertEqual(to_bytes("\xa3 49"), b"\xc2\xa3 49")
 
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
-        self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
+        self.assertEqual(to_bytes("\xa3 49", "latin-1"), b"\xa3 49")
 
     def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
-        self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
+        self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e")
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_bytes, unittest)
 
     def test_errors_argument(self):
-        self.assertEqual(
-            to_bytes('a\ufffdb', 'latin-1', errors='replace'),
-            b'a?b'
-        )
+        self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b")
 
 
 class MemoizedMethodTest(unittest.TestCase):
     def test_memoizemethod_noargs(self):
         class A:
-
             @memoizemethod_noargs
             def cached(self):
                 return object()
@@ -141,7 +141,7 @@ class BinaryIsTextTest(unittest.TestCase):
         assert binary_is_text(b"hello")
 
     def test_utf_16_strings_contain_null_bytes(self):
-        assert binary_is_text("hello".encode('utf-16'))
+        assert binary_is_text("hello".encode("utf-16"))
 
     def test_one_with_encoding(self):
         assert binary_is_text(b"<div>Price \xa3</div>")
@@ -151,7 +151,6 @@ class BinaryIsTextTest(unittest.TestCase):
 
 
 class UtilsPythonTestCase(unittest.TestCase):
-
     def test_equal_attributes(self):
         class Obj:
             pass
@@ -161,44 +160,44 @@ class UtilsPythonTestCase(unittest.TestCase):
         # no attributes given return False
         self.assertFalse(equal_attributes(a, b, []))
         # not existent attributes
-        self.assertFalse(equal_attributes(a, b, ['x', 'y']))
+        self.assertFalse(equal_attributes(a, b, ["x", "y"]))
 
         a.x = 1
         b.x = 1
         # equal attribute
-        self.assertTrue(equal_attributes(a, b, ['x']))
+        self.assertTrue(equal_attributes(a, b, ["x"]))
 
         b.y = 2
         # obj1 has no attribute y
-        self.assertFalse(equal_attributes(a, b, ['x', 'y']))
+        self.assertFalse(equal_attributes(a, b, ["x", "y"]))
 
         a.y = 2
         # equal attributes
-        self.assertTrue(equal_attributes(a, b, ['x', 'y']))
+        self.assertTrue(equal_attributes(a, b, ["x", "y"]))
 
         a.y = 1
         # differente attributes
-        self.assertFalse(equal_attributes(a, b, ['x', 'y']))
+        self.assertFalse(equal_attributes(a, b, ["x", "y"]))
 
         # test callable
         a.meta = {}
         b.meta = {}
-        self.assertTrue(equal_attributes(a, b, ['meta']))
+        self.assertTrue(equal_attributes(a, b, ["meta"]))
 
         # compare ['meta']['a']
-        a.meta['z'] = 1
-        b.meta['z'] = 1
+        a.meta["z"] = 1
+        b.meta["z"] = 1
 
-        get_z = operator.itemgetter('z')
-        get_meta = operator.attrgetter('meta')
+        get_z = operator.itemgetter("z")
+        get_meta = operator.attrgetter("meta")
 
         def compare_z(obj):
             return get_z(get_meta(obj))
 
-        self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
+        self.assertTrue(equal_attributes(a, b, [compare_z, "x"]))
         # fail z equality
-        a.meta['z'] = 2
-        self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
+        a.meta["z"] = 2
+        self.assertFalse(equal_attributes(a, b, [compare_z, "x"]))
 
     def test_get_func_args(self):
         def f1(a, b, c):
@@ -218,7 +217,6 @@ class UtilsPythonTestCase(unittest.TestCase):
                 pass
 
         class Callable:
-
             def __call__(self, a, b, c):
                 pass
 
@@ -228,33 +226,38 @@ class UtilsPythonTestCase(unittest.TestCase):
         partial_f2 = functools.partial(f1, b=None)
         partial_f3 = functools.partial(partial_f2, None)
 
-        self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
-        self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
-        self.assertEqual(get_func_args(f3), ['a', 'b', 'c'])
-        self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
-        self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
-        self.assertEqual(get_func_args(partial_f1), ['b', 'c'])
-        self.assertEqual(get_func_args(partial_f2), ['a', 'c'])
-        self.assertEqual(get_func_args(partial_f3), ['c'])
-        self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
+        self.assertEqual(get_func_args(f1), ["a", "b", "c"])
+        self.assertEqual(get_func_args(f2), ["a", "b", "c"])
+        self.assertEqual(get_func_args(f3), ["a", "b", "c"])
+        self.assertEqual(get_func_args(A), ["a", "b", "c"])
+        self.assertEqual(get_func_args(a.method), ["a", "b", "c"])
+        self.assertEqual(get_func_args(partial_f1), ["b", "c"])
+        self.assertEqual(get_func_args(partial_f2), ["a", "c"])
+        self.assertEqual(get_func_args(partial_f3), ["c"])
+        self.assertEqual(get_func_args(cal), ["a", "b", "c"])
         self.assertEqual(get_func_args(object), [])
 
-        if platform.python_implementation() == 'CPython':
+        if platform.python_implementation() == "CPython":
             # TODO: how do we fix this to return the actual argument names?
             self.assertEqual(get_func_args(str.split), [])
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
-        elif platform.python_implementation() == 'PyPy':
-            self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
-            self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
+        elif platform.python_implementation() == "PyPy":
+            self.assertEqual(
+                get_func_args(str.split, stripself=True), ["sep", "maxsplit"]
+            )
+            self.assertEqual(
+                get_func_args(operator.itemgetter(2), stripself=True), ["obj"]
+            )
+            self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
         self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
         self.assertEqual(
-            without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
-            {'one': 1, 'three': 3, 'four': 4})
+            without_none_values({"one": 1, "none": None, "three": 3, "four": 4}),
+            {"one": 1, "three": 3, "four": 4},
+        )
 
 
 if __name__ == "__main__":
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index a92d9a0ac..4760bf92b 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -23,24 +23,32 @@ from scrapy.utils.test import get_crawler
 
 
 class UtilsRequestTest(unittest.TestCase):
-
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
-        request_authenticate(r, 'someuser', 'somepass')
-        self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')
+        request_authenticate(r, "someuser", "somepass")
+        self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=")
 
     def test_request_httprepr(self):
         r1 = Request("http://www.example.com")
-        self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+        self.assertEqual(
+            request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
+        )
 
         r1 = Request("http://www.example.com/some/page.html?arg=1")
-        self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
-
-        r1 = Request("http://www.example.com", method='POST',
-                     headers={"Content-type": b"text/html"}, body=b"Some body")
         self.assertEqual(
             request_httprepr(r1),
-            b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
+            b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+        )
+
+        r1 = Request(
+            "http://www.example.com",
+            method="POST",
+            headers={"Content-type": b"text/html"},
+            body=b"Some body",
+        )
+        self.assertEqual(
+            request_httprepr(r1),
+            b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
         )
 
     def test_request_httprepr_for_non_http_request(self):
@@ -61,77 +69,77 @@ class FingerprintTest(unittest.TestCase):
     known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
         (
             Request("http://example.org"),
-            b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
+            b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",
             {},
         ),
         (
             Request("https://example.org"),
-            b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+            b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l",
             {},
         ),
         (
             Request("https://example.org?a"),
-            b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
+            b"G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c",
             {},
         ),
         (
             Request("https://example.org?a=b"),
-            b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
+            b"\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17",
             {},
         ),
         (
             Request("https://example.org?a=b&a"),
-            b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
+            b"t+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96",
             {},
         ),
         (
             Request("https://example.org?a=b&a=c"),
-            b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
+            b"\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef",
             {},
         ),
         (
-            Request("https://example.org", method='POST'),
-            b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
+            Request("https://example.org", method="POST"),
+            b"\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d",
             {},
         ),
         (
-            Request("https://example.org", body=b'a'),
-            b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
+            Request("https://example.org", body=b"a"),
+            b"\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_",
             {},
         ),
         (
-            Request("https://example.org", method='POST', body=b'a'),
-            b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
+            Request("https://example.org", method="POST", body=b"a"),
+            b"5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7",
             {},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+            Request("https://example.org#a", headers={"A": b"B"}),
+            b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l",
             {},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
-            {'include_headers': ['A']},
+            Request("https://example.org#a", headers={"A": b"B"}),
+            b"]\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.",
+            {"include_headers": ["A"]},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
-            {'keep_fragments': True},
+            Request("https://example.org#a", headers={"A": b"B"}),
+            b"<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f",
+            {"keep_fragments": True},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
-            {'include_headers': ['A'], 'keep_fragments': True},
+            Request("https://example.org#a", headers={"A": b"B"}),
+            b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11",
+            {"include_headers": ["A"], "keep_fragments": True},
         ),
         (
             Request("https://example.org/ab"),
-            b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
+            b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6",
             {},
         ),
         (
-            Request("https://example.org/a", body=b'b'),
-            b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
+            Request("https://example.org/a", body=b"b"),
+            b"_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5",
             {},
         ),
     )
@@ -143,42 +151,42 @@ class FingerprintTest(unittest.TestCase):
         self.assertEqual(self.function(r1), self.function(r2))
 
     def test_query_string_key_without_value(self):
-        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
-        r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199")
+        r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
         self.assertNotEqual(self.function(r1), self.function(r2))
 
     def test_caching(self):
-        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
-        self.assertEqual(
-            self.function(r1),
-            self.cache[r1][self.default_cache_key]
-        )
+        r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
+        self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key])
 
     def test_header(self):
         r1 = Request("http://www.example.com/members/offers.html")
         r2 = Request("http://www.example.com/members/offers.html")
-        r2.headers['SESSIONID'] = b"somehash"
+        r2.headers["SESSIONID"] = b"somehash"
         self.assertEqual(self.function(r1), self.function(r2))
 
     def test_headers(self):
         r1 = Request("http://www.example.com/")
         r2 = Request("http://www.example.com/")
-        r2.headers['Accept-Language'] = b'en'
+        r2.headers["Accept-Language"] = b"en"
         r3 = Request("http://www.example.com/")
-        r3.headers['Accept-Language'] = b'en'
-        r3.headers['SESSIONID'] = b"somehash"
+        r3.headers["Accept-Language"] = b"en"
+        r3.headers["SESSIONID"] = b"somehash"
 
         self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
 
-        self.assertEqual(self.function(r1),
-                         self.function(r1, include_headers=['Accept-Language']))
+        self.assertEqual(
+            self.function(r1), self.function(r1, include_headers=["Accept-Language"])
+        )
 
         self.assertNotEqual(
-            self.function(r1),
-            self.function(r2, include_headers=['Accept-Language']))
+            self.function(r1), self.function(r2, include_headers=["Accept-Language"])
+        )
 
-        self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
-                         self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
+        self.assertEqual(
+            self.function(r3, include_headers=["accept-language", "sessionid"]),
+            self.function(r3, include_headers=["SESSIONID", "Accept-Language"]),
+        )
 
     def test_fragment(self):
         r1 = Request("http://www.example.com/test.html")
@@ -190,8 +198,8 @@ class FingerprintTest(unittest.TestCase):
 
     def test_method_and_body(self):
         r1 = Request("http://www.example.com")
-        r2 = Request("http://www.example.com", method='POST')
-        r3 = Request("http://www.example.com", method='POST', body=b'request body')
+        r2 = Request("http://www.example.com", method="POST")
+        r3 = Request("http://www.example.com", method="POST", body=b"request body")
 
         self.assertNotEqual(self.function(r1), self.function(r2))
         self.assertNotEqual(self.function(r2), self.function(r3))
@@ -209,7 +217,7 @@ class FingerprintTest(unittest.TestCase):
         # would put the body right after the URL.
         r1 = Request("http://www.example.com/foo")
         fp1 = self.function(r1)
-        r2 = Request("http://www.example.com/f", body=b'oo')
+        r2 = Request("http://www.example.com/f", body=b"oo")
         fp2 = self.function(r2)
         self.assertNotEqual(fp1, fp2)
 
@@ -217,13 +225,9 @@ class FingerprintTest(unittest.TestCase):
         """Test hardcoded hashes, to make sure future changes to not introduce
         backward incompatibilities."""
         actual = [
-            self.function(request, **kwargs)
-            for request, _, kwargs in self.known_hashes
-        ]
-        expected = [
-            _fingerprint
-            for _, _fingerprint, _ in self.known_hashes
+            self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
         ]
+        expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes]
         self.assertEqual(actual, expected)
 
 
@@ -233,77 +237,77 @@ class RequestFingerprintTest(FingerprintTest):
     known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
         (
             Request("http://example.org"),
-            'b2e5245ef826fd9576c93bd6e392fce3133fab62',
+            "b2e5245ef826fd9576c93bd6e392fce3133fab62",
             {},
         ),
         (
             Request("https://example.org"),
-            'bd10a0a89ea32cdee77917320f1309b0da87e892',
+            "bd10a0a89ea32cdee77917320f1309b0da87e892",
             {},
         ),
         (
             Request("https://example.org?a"),
-            '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
+            "2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
             {},
         ),
         (
             Request("https://example.org?a=b"),
-            '42e5fe149b147476e3f67ad0670c57b4cc57856a',
+            "42e5fe149b147476e3f67ad0670c57b4cc57856a",
             {},
         ),
         (
             Request("https://example.org?a=b&a"),
-            'd23a9787cb56c6375c2cae4453c5a8c634526942',
+            "d23a9787cb56c6375c2cae4453c5a8c634526942",
             {},
         ),
         (
             Request("https://example.org?a=b&a=c"),
-            '9a18a7a8552a9182b7f1e05d33876409e421e5c5',
+            "9a18a7a8552a9182b7f1e05d33876409e421e5c5",
             {},
         ),
         (
-            Request("https://example.org", method='POST'),
-            'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
+            Request("https://example.org", method="POST"),
+            "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
             {},
         ),
         (
-            Request("https://example.org", body=b'a'),
-            '4bb136e54e715a4ea7a9dd1101831765d33f2d60',
+            Request("https://example.org", body=b"a"),
+            "4bb136e54e715a4ea7a9dd1101831765d33f2d60",
             {},
         ),
         (
-            Request("https://example.org", method='POST', body=b'a'),
-            '6c6595374a304b293be762f7b7be3f54e9947c65',
+            Request("https://example.org", method="POST", body=b"a"),
+            "6c6595374a304b293be762f7b7be3f54e9947c65",
             {},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            'bd10a0a89ea32cdee77917320f1309b0da87e892',
+            Request("https://example.org#a", headers={"A": b"B"}),
+            "bd10a0a89ea32cdee77917320f1309b0da87e892",
             {},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            '515b633cb3ca502a33a9d8c890e889ec1e425e65',
-            {'include_headers': ['A']},
+            Request("https://example.org#a", headers={"A": b"B"}),
+            "515b633cb3ca502a33a9d8c890e889ec1e425e65",
+            {"include_headers": ["A"]},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            '505c96e7da675920dfef58725e8c957dfdb38f47',
-            {'keep_fragments': True},
+            Request("https://example.org#a", headers={"A": b"B"}),
+            "505c96e7da675920dfef58725e8c957dfdb38f47",
+            {"keep_fragments": True},
         ),
         (
-            Request("https://example.org#a", headers={'A': b'B'}),
-            'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
-            {'include_headers': ['A'], 'keep_fragments': True},
+            Request("https://example.org#a", headers={"A": b"B"}),
+            "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
+            {"include_headers": ["A"], "keep_fragments": True},
         ),
         (
             Request("https://example.org/ab"),
-            '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+            "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
             {},
         ),
         (
-            Request("https://example.org/a", body=b'b'),
-            '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+            Request("https://example.org/a", body=b"b"),
+            "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
             {},
         ),
     )
@@ -314,36 +318,29 @@ class RequestFingerprintTest(FingerprintTest):
     def tearDown(self) -> None:
         warnings.simplefilter("default", ScrapyDeprecationWarning)
 
-    @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+    @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
     def test_part_separation(self):
         super().test_part_separation()
 
 
 class RequestFingerprintDeprecationTest(unittest.TestCase):
-
     def test_deprecation_default_parameters(self):
         with pytest.warns(ScrapyDeprecationWarning) as warnings:
             request_fingerprint(Request("http://www.example.com"))
         messages = [str(warning.message) for warning in warnings]
         self.assertTrue(
-            any(
-                'Call to deprecated function' in message
-                for message in messages
-            )
+            any("Call to deprecated function" in message for message in messages)
         )
-        self.assertFalse(any('non-default' in message for message in messages))
+        self.assertFalse(any("non-default" in message for message in messages))
 
     def test_deprecation_non_default_parameters(self):
         with pytest.warns(ScrapyDeprecationWarning) as warnings:
             request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
         messages = [str(warning.message) for warning in warnings]
         self.assertTrue(
-            any(
-                'Call to deprecated function' in message
-                for message in messages
-            )
+            any("Call to deprecated function" in message for message in messages)
         )
-        self.assertTrue(any('non-default' in message for message in messages))
+        self.assertTrue(any("non-default" in message for message in messages))
 
 
 class RequestFingerprintAsBytesTest(FingerprintTest):
@@ -352,24 +349,21 @@ class RequestFingerprintAsBytesTest(FingerprintTest):
     known_hashes = RequestFingerprintTest.known_hashes
 
     def test_caching(self):
-        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
         self.assertEqual(
-            self.function(r1),
-            bytes.fromhex(self.cache[r1][self.default_cache_key])
+            self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
         )
 
-    @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+    @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
     def test_part_separation(self):
         super().test_part_separation()
 
     def test_hashes(self):
         actual = [
-            self.function(request, **kwargs)
-            for request, _, kwargs in self.known_hashes
+            self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
         ]
         expected = [
-            bytes.fromhex(_fingerprint)
-            for _, _fingerprint, _ in self.known_hashes
+            bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
         ]
         self.assertEqual(actual, expected)
 
@@ -385,8 +379,10 @@ def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False)
     if cache_key not in cache:
         fp = sha1()
         fp.update(to_bytes(request.method))
-        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
-        fp.update(request.body or b'')
+        fp.update(
+            to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
+        )
+        fp.update(request.body or b"")
         if include_headers:
             for hdr in include_headers:
                 if hdr in request.headers:
@@ -401,39 +397,38 @@ REQUEST_OBJECTS_TO_TEST = (
     Request("http://www.example.com/"),
     Request("http://www.example.com/query?id=111&cat=222"),
     Request("http://www.example.com/query?cat=222&id=111"),
-    Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
-    Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
+    Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
+    Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
     Request("http://www.example.com/members/offers.html"),
     Request(
         "http://www.example.com/members/offers.html",
-        headers={'SESSIONID': b"somehash"},
+        headers={"SESSIONID": b"somehash"},
     ),
     Request(
         "http://www.example.com/",
-        headers={'Accept-Language': b"en"},
+        headers={"Accept-Language": b"en"},
     ),
     Request(
         "http://www.example.com/",
         headers={
-            'Accept-Language': b"en",
-            'SESSIONID': b"somehash",
+            "Accept-Language": b"en",
+            "SESSIONID": b"somehash",
         },
     ),
     Request("http://www.example.com/test.html"),
     Request("http://www.example.com/test.html#fragment"),
-    Request("http://www.example.com", method='POST'),
-    Request("http://www.example.com", method='POST', body=b'request body'),
+    Request("http://www.example.com", method="POST"),
+    Request("http://www.example.com", method="POST", body=b"request body"),
 )
 
 
 class BackwardCompatibilityTestCase(unittest.TestCase):
-
     def test_function_backward_compatibility(self):
         include_headers_to_test = (
             None,
-            ['Accept-Language'],
-            ['accept-language', 'sessionid'],
-            ['SESSIONID', 'Accept-Language'],
+            ["Accept-Language"],
+            ["accept-language", "sessionid"],
+            ["SESSIONID", "Accept-Language"],
         )
         for request_object in REQUEST_OBJECTS_TO_TEST:
             for include_headers in include_headers_to_test:
@@ -475,14 +470,14 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
                     fp = sha1()
                     fp.update(to_bytes(request.method))
                     fp.update(to_bytes(canonicalize_url(request.url)))
-                    fp.update(request.body or b'')
+                    fp.update(request.body or b"")
                     self.cache[request] = fp.digest()
                 return self.cache[request]
 
         for request_object in REQUEST_OBJECTS_TO_TEST:
             with warnings.catch_warnings() as logged_warnings:
                 settings = {
-                    'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+                    "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
                 }
                 crawler = get_crawler(settings_dict=settings)
                 fp = crawler.request_fingerprinter.fingerprint(request_object)
@@ -492,11 +487,10 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
 
 
 class RequestFingerprinterTestCase(unittest.TestCase):
-
     def test_default_implementation(self):
         with warnings.catch_warnings(record=True) as logged_warnings:
             crawler = get_crawler(prevent_warnings=False)
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         self.assertEqual(
             crawler.request_fingerprinter.fingerprint(request),
             _request_fingerprint_as_bytes(request),
@@ -505,11 +499,11 @@ class RequestFingerprinterTestCase(unittest.TestCase):
 
     def test_deprecated_implementation(self):
         settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6',
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
         }
         with warnings.catch_warnings(record=True) as logged_warnings:
             crawler = get_crawler(settings_dict=settings)
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         self.assertEqual(
             crawler.request_fingerprinter.fingerprint(request),
             _request_fingerprint_as_bytes(request),
@@ -518,11 +512,11 @@ class RequestFingerprinterTestCase(unittest.TestCase):
 
     def test_recommended_implementation(self):
         settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
         }
         with warnings.catch_warnings(record=True) as logged_warnings:
             crawler = get_crawler(settings_dict=settings)
-        request = Request('https://example.com')
+        request = Request("https://example.com")
         self.assertEqual(
             crawler.request_fingerprinter.fingerprint(request),
             fingerprint(request),
@@ -531,34 +525,30 @@ class RequestFingerprinterTestCase(unittest.TestCase):
 
     def test_unknown_implementation(self):
         settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
+            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
         }
         with self.assertRaises(ValueError):
             get_crawler(settings_dict=settings)
 
 
 class CustomRequestFingerprinterTestCase(unittest.TestCase):
-
     def test_include_headers(self):
-
         class RequestFingerprinter:
-
             def fingerprint(self, request):
-                return fingerprint(request, include_headers=['X-ID'])
+                return fingerprint(request, include_headers=["X-ID"])
 
         settings = {
-            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
         }
         crawler = get_crawler(settings_dict=settings)
 
-        r1 = Request("http://www.example.com", headers={'X-ID': '1'})
+        r1 = Request("http://www.example.com", headers={"X-ID": "1"})
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
-        r2 = Request("http://www.example.com", headers={'X-ID': '2'})
+        r2 = Request("http://www.example.com", headers={"X-ID": "2"})
         fp2 = crawler.request_fingerprinter.fingerprint(r2)
         self.assertNotEqual(fp1, fp2)
 
     def test_dont_canonicalize(self):
-
         class RequestFingerprinter:
             cache = WeakKeyDictionary()
 
@@ -570,7 +560,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
                 return self.cache[request]
 
         settings = {
-            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
         }
         crawler = get_crawler(settings_dict=settings)
 
@@ -581,26 +571,24 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
         self.assertNotEqual(fp1, fp2)
 
     def test_meta(self):
-
         class RequestFingerprinter:
-
             def fingerprint(self, request):
-                if 'fingerprint' in request.meta:
-                    return request.meta['fingerprint']
+                if "fingerprint" in request.meta:
+                    return request.meta["fingerprint"]
                 return fingerprint(request)
 
         settings = {
-            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
         }
         crawler = get_crawler(settings_dict=settings)
 
         r1 = Request("http://www.example.com")
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
-        r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+        r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
         fp2 = crawler.request_fingerprinter.fingerprint(r2)
-        r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+        r3 = Request("http://www.example.com", meta={"fingerprint": "a"})
         fp3 = crawler.request_fingerprinter.fingerprint(r3)
-        r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
+        r4 = Request("http://www.example.com", meta={"fingerprint": "b"})
         fp4 = crawler.request_fingerprinter.fingerprint(r4)
         self.assertNotEqual(fp1, fp2)
         self.assertNotEqual(fp1, fp4)
@@ -608,55 +596,50 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
         self.assertEqual(fp2, fp3)
 
     def test_from_crawler(self):
-
         class RequestFingerprinter:
-
             @classmethod
             def from_crawler(cls, crawler):
                 return cls(crawler)
 
             def __init__(self, crawler):
-                self._fingerprint = crawler.settings['FINGERPRINT']
+                self._fingerprint = crawler.settings["FINGERPRINT"]
 
             def fingerprint(self, request):
                 return self._fingerprint
 
         settings = {
-            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
-            'FINGERPRINT': b'fingerprint',
+            "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
+            "FINGERPRINT": b"fingerprint",
         }
         crawler = get_crawler(settings_dict=settings)
 
         request = Request("http://www.example.com")
         fingerprint = crawler.request_fingerprinter.fingerprint(request)
-        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+        self.assertEqual(fingerprint, settings["FINGERPRINT"])
 
     def test_from_settings(self):
-
         class RequestFingerprinter:
-
             @classmethod
             def from_settings(cls, settings):
                 return cls(settings)
 
             def __init__(self, settings):
-                self._fingerprint = settings['FINGERPRINT']
+                self._fingerprint = settings["FINGERPRINT"]
 
             def fingerprint(self, request):
                 return self._fingerprint
 
         settings = {
-            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
-            'FINGERPRINT': b'fingerprint',
+            "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
+            "FINGERPRINT": b"fingerprint",
         }
         crawler = get_crawler(settings_dict=settings)
 
         request = Request("http://www.example.com")
         fingerprint = crawler.request_fingerprinter.fingerprint(request)
-        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+        self.assertEqual(fingerprint, settings["FINGERPRINT"])
 
     def test_from_crawler_and_settings(self):
-
         class RequestFingerprinter:
 
             # This method is ignored due to the presence of from_crawler
@@ -669,20 +652,20 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
                 return cls(crawler)
 
             def __init__(self, crawler):
-                self._fingerprint = crawler.settings['FINGERPRINT']
+                self._fingerprint = crawler.settings["FINGERPRINT"]
 
             def fingerprint(self, request):
                 return self._fingerprint
 
         settings = {
-            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
-            'FINGERPRINT': b'fingerprint',
+            "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
+            "FINGERPRINT": b"fingerprint",
         }
         crawler = get_crawler(settings_dict=settings)
 
         request = Request("http://www.example.com")
         fingerprint = crawler.request_fingerprinter.fingerprint(request)
-        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+        self.assertEqual(fingerprint, settings["FINGERPRINT"])
 
 
 if __name__ == "__main__":
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index cdf972933..d82aa19c6 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -6,32 +6,49 @@ from urllib.parse import urlparse
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, HtmlResponse
 from scrapy.utils.python import to_bytes
-from scrapy.utils.response import (response_httprepr, open_in_browser,
-                                   get_meta_refresh, get_base_url, response_status_message)
+from scrapy.utils.response import (
+    response_httprepr,
+    open_in_browser,
+    get_meta_refresh,
+    get_base_url,
+    response_status_message,
+)
 
 
-__doctests__ = ['scrapy.utils.response']
+__doctests__ = ["scrapy.utils.response"]
 
 
 class ResponseUtilsTest(unittest.TestCase):
-    dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
+    dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
 
     def test_response_httprepr(self):
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
 
             r1 = Response("http://www.example.com")
-            self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
+            self.assertEqual(response_httprepr(r1), b"HTTP/1.1 200 OK\r\n\r\n")
 
-            r1 = Response("http://www.example.com", status=404,
-                          headers={"Content-type": "text/html"}, body=b"Some body")
-            self.assertEqual(response_httprepr(r1),
-                             b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+            r1 = Response(
+                "http://www.example.com",
+                status=404,
+                headers={"Content-type": "text/html"},
+                body=b"Some body",
+            )
+            self.assertEqual(
+                response_httprepr(r1),
+                b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body",
+            )
 
-            r1 = Response("http://www.example.com", status=6666,
-                          headers={"Content-type": "text/html"}, body=b"Some body")
-            self.assertEqual(response_httprepr(r1),
-                             b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
+            r1 = Response(
+                "http://www.example.com",
+                status=6666,
+                headers={"Content-type": "text/html"},
+                body=b"Some body",
+            )
+            self.assertEqual(
+                response_httprepr(r1),
+                b"HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body",
+            )
 
     def test_open_in_browser(self):
         url = "http:///www.example.com/some/page.html"
@@ -40,10 +57,11 @@ class ResponseUtilsTest(unittest.TestCase):
         def browser_open(burl):
             path = urlparse(burl).path
             if not path or not Path(path).exists():
-                path = burl.replace('file://', '')
+                path = burl.replace("file://", "")
             bbody = Path(path).read_bytes()
             self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
             return True
+
         response = HtmlResponse(url, body=body)
         assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
 
@@ -51,45 +69,60 @@ class ResponseUtilsTest(unittest.TestCase):
         self.assertRaises(TypeError, open_in_browser, resp, debug=True)
 
     def test_get_meta_refresh(self):
-        r1 = HtmlResponse("http://www.example.com", body=b"""
+        r1 = HtmlResponse(
+            "http://www.example.com",
+            body=b"""
         <html>
         <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
         <body>blahablsdfsal&amp;</body>
-        </html>""")
-        r2 = HtmlResponse("http://www.example.com", body=b"""
+        </html>""",
+        )
+        r2 = HtmlResponse(
+            "http://www.example.com",
+            body=b"""
         <html>
         <head><title>Dummy</title><noScript>
         <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
         </noSCRIPT>
         <body>blahablsdfsal&amp;</body>
-        </html>""")
-        r3 = HtmlResponse("http://www.example.com", body=b"""
+        </html>""",
+        )
+        r3 = HtmlResponse(
+            "http://www.example.com",
+            body=b"""
     <noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
     <script type="text/javascript">
     if(!checkCookies()){
         document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');
     }
     </script>
-        """)
-        self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
+        """,
+        )
+        self.assertEqual(get_meta_refresh(r1), (5.0, "http://example.org/newpage"))
         self.assertEqual(get_meta_refresh(r2), (None, None))
         self.assertEqual(get_meta_refresh(r3), (None, None))
 
     def test_get_base_url(self):
-        resp = HtmlResponse("http://www.example.com", body=b"""
+        resp = HtmlResponse(
+            "http://www.example.com",
+            body=b"""
         <html>
         <head><base href="http://www.example.com/img/" target="_blank"></head>
         <body>blahablsdfsal&amp;</body>
-        </html>""")
+        </html>""",
+        )
         self.assertEqual(get_base_url(resp), "http://www.example.com/img/")
 
-        resp2 = HtmlResponse("http://www.example.com", body=b"""
-        <html><body>blahablsdfsal&amp;</body></html>""")
+        resp2 = HtmlResponse(
+            "http://www.example.com",
+            body=b"""
+        <html><body>blahablsdfsal&amp;</body></html>""",
+        )
         self.assertEqual(get_base_url(resp2), "http://www.example.com")
 
     def test_response_status_message(self):
-        self.assertEqual(response_status_message(200), '200 OK')
-        self.assertEqual(response_status_message(404), '404 Not Found')
+        self.assertEqual(response_status_message(200), "200 OK")
+        self.assertEqual(response_status_message(404), "404 Not Found")
         self.assertEqual(response_status_message(573), "573 Unknown Status")
 
     def test_inject_base_url(self):
@@ -98,36 +131,50 @@ class ResponseUtilsTest(unittest.TestCase):
         def check_base_url(burl):
             path = urlparse(burl).path
             if not path or not Path(path).exists():
-                path = burl.replace('file://', '')
+                path = burl.replace("file://", "")
             bbody = Path(path).read_bytes()
             self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
             return True
 
-        r1 = HtmlResponse(url, body=b"""
+        r1 = HtmlResponse(
+            url,
+            body=b"""
         <html>
             <head><title>Dummy</title></head>
             <body><p>Hello world.</p></body>
-        </html>""")
-        r2 = HtmlResponse(url, body=b"""
+        </html>""",
+        )
+        r2 = HtmlResponse(
+            url,
+            body=b"""
         <html>
             <head id="foo"><title>Dummy</title></head>
             <body>Hello world.</body>
-        </html>""")
-        r3 = HtmlResponse(url, body=b"""
+        </html>""",
+        )
+        r3 = HtmlResponse(
+            url,
+            body=b"""
         <html>
             <head><title>Dummy</title></head>
             <body>
                 <header>Hello header</header>
                 <p>Hello world.</p>
             </body>
-        </html>""")
-        r4 = HtmlResponse(url, body=b"""
+        </html>""",
+        )
+        r4 = HtmlResponse(
+            url,
+            body=b"""
         <html>
             <!-- <head>Dummy comment</head> -->
             <head><title>Dummy</title></head>
             <body><p>Hello world.</p></body>
-        </html>""")
-        r5 = HtmlResponse(url, body=b"""
+        </html>""",
+        )
+        r5 = HtmlResponse(
+            url,
+            body=b"""
         <html>
             <!--[if IE]>
             <head><title>IE head</title></head>
@@ -136,10 +183,19 @@ class ResponseUtilsTest(unittest.TestCase):
             <head><title>Standard head</title></head>
             <!--<![endif]-->
             <body><p>Hello world.</p></body>
-        </html>""")
+        </html>""",
+        )
 
         assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
-        assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
-        assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
-        assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
-        assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"
+        assert open_in_browser(
+            r2, _openfunc=check_base_url
+        ), "Inject base url with argumented head"
+        assert open_in_browser(
+            r3, _openfunc=check_base_url
+        ), "Inject unique base url with misleading tag"
+        assert open_in_browser(
+            r4, _openfunc=check_base_url
+        ), "Inject unique base url with misleading comment"
+        assert open_in_browser(
+            r5, _openfunc=check_base_url
+        ), "Inject unique base url with conditional comment"
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index a51de1877..20aebc2d7 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -12,7 +12,6 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
 class JsonEncoderTestCase(unittest.TestCase):
-
     def setUp(self):
         self.encoder = ScrapyJSONEncoder(sort_keys=True)
 
@@ -25,19 +24,27 @@ class JsonEncoderTestCase(unittest.TestCase):
         ts = "10:11:12"
         dec = Decimal("1000.12")
         decs = "1000.12"
-        s = {'foo'}
-        ss = ['foo']
+        s = {"foo"}
+        ss = ["foo"]
         dt_set = {dt}
         dt_sets = [dts]
 
-        for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts),
-                              (dec, decs), (['foo', d], ['foo', ds]), (s, ss),
-                              (dt_set, dt_sets)]:
-            self.assertEqual(self.encoder.encode(input),
-                             json.dumps(output, sort_keys=True))
+        for input, output in [
+            ("foo", "foo"),
+            (d, ds),
+            (t, ts),
+            (dt, dts),
+            (dec, decs),
+            (["foo", d], ["foo", ds]),
+            (s, ss),
+            (dt_set, dt_sets),
+        ]:
+            self.assertEqual(
+                self.encoder.encode(input), json.dumps(output, sort_keys=True)
+            )
 
     def test_encode_deferred(self):
-        self.assertIn('Deferred', self.encoder.encode(defer.Deferred()))
+        self.assertIn("Deferred", self.encoder.encode(defer.Deferred()))
 
     def test_encode_request(self):
         r = Request("http://www.example.com/lala")
@@ -61,8 +68,7 @@ class JsonEncoderTestCase(unittest.TestCase):
         item = TestDataClass(name="Product", url="http://product.org", price=1)
         encoded = self.encoder.encode(item)
         self.assertEqual(
-            encoded,
-            '{"name": "Product", "price": 1, "url": "http://product.org"}'
+            encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}'
         )
 
     def test_encode_attrs_item(self):
@@ -75,6 +81,5 @@ class JsonEncoderTestCase(unittest.TestCase):
         item = AttrsItem(name="Product", url="http://product.org", price=1)
         encoded = self.encoder.encode(item)
         self.assertEqual(
-            encoded,
-            '{"name": "Product", "price": 1, "url": "http://product.org"}'
+            encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}'
         )
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index a36e7bc97..65b99e0c4 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -12,7 +12,6 @@ from scrapy.utils.test import get_from_asyncio_queue
 
 
 class SendCatchLogTest(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_send_catch_log(self):
         test_signal = object()
@@ -22,16 +21,18 @@ class SendCatchLogTest(unittest.TestCase):
         dispatcher.connect(self.ok_handler, signal=test_signal)
         with LogCapture() as log:
             result = yield defer.maybeDeferred(
-                self._get_result, test_signal, arg='test',
-                handlers_called=handlers_called
+                self._get_result,
+                test_signal,
+                arg="test",
+                handlers_called=handlers_called,
             )
 
         assert self.error_handler in handlers_called
         assert self.ok_handler in handlers_called
         self.assertEqual(len(log.records), 1)
         record = log.records[0]
-        self.assertIn('error_handler', record.getMessage())
-        self.assertEqual(record.levelname, 'ERROR')
+        self.assertIn("error_handler", record.getMessage())
+        self.assertEqual(record.levelname, "ERROR")
         self.assertEqual(result[0][0], self.error_handler)
         self.assertIsInstance(result[0][1], Failure)
         self.assertEqual(result[1], (self.ok_handler, "OK"))
@@ -48,32 +49,29 @@ class SendCatchLogTest(unittest.TestCase):
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
-        assert arg == 'test'
+        assert arg == "test"
         return "OK"
 
 
 class SendCatchLogDeferredTest(SendCatchLogTest):
-
     def _get_result(self, signal, *a, **kw):
         return send_catch_log_deferred(signal, *a, **kw)
 
 
 class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
-
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
-        assert arg == 'test'
+        assert arg == "test"
         d = defer.Deferred()
         reactor.callLater(0, d.callback, "OK")
         return d
 
 
-@mark.usefixtures('reactor_pytest')
+@mark.usefixtures("reactor_pytest")
 class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
-
     async def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
-        assert arg == 'test'
+        assert arg == "test"
         await defer.succeed(42)
         return "OK"
 
@@ -83,10 +81,9 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
 
 @mark.only_asyncio()
 class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
-
     async def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
-        assert arg == 'test'
+        assert arg == "test"
         await asyncio.sleep(0.2)
         return await get_from_asyncio_queue("OK")
 
@@ -95,7 +92,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
 
 
 class SendCatchLogTest2(unittest.TestCase):
-
     def test_error_logged_if_deferred_not_supported(self):
         def test_handler():
             return defer.Deferred()
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 23eb261b7..ce0de0722 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -4,9 +4,9 @@ from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 
 
 class SitemapTest(unittest.TestCase):
-
     def test_sitemap(self):
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
   <url>
     <loc>http://www.example.com/</loc>
@@ -20,19 +20,30 @@ class SitemapTest(unittest.TestCase):
     <changefreq>weekly</changefreq>
     <priority>0.8</priority>
   </url>
-</urlset>""")
-        assert s.type == 'urlset'
+</urlset>"""
+        )
+        assert s.type == "urlset"
         self.assertEqual(
             list(s),
             [
-                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-                {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html',
-                 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
-            ]
+                {
+                    "priority": "1",
+                    "loc": "http://www.example.com/",
+                    "lastmod": "2009-08-16",
+                    "changefreq": "daily",
+                },
+                {
+                    "priority": "0.8",
+                    "loc": "http://www.example.com/Special-Offers.html",
+                    "lastmod": "2009-08-16",
+                    "changefreq": "weekly",
+                },
+            ],
         )
 
     def test_sitemap_index(self):
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
    <sitemap>
       <loc>http://www.example.com/sitemap1.xml.gz</loc>
@@ -42,21 +53,29 @@ class SitemapTest(unittest.TestCase):
       <loc>http://www.example.com/sitemap2.xml.gz</loc>
       <lastmod>2005-01-01</lastmod>
    </sitemap>
-</sitemapindex>""")
-        assert s.type == 'sitemapindex'
+</sitemapindex>"""
+        )
+        assert s.type == "sitemapindex"
         self.assertEqual(
             list(s),
             [
-                {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'},
-                {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'},
-            ]
+                {
+                    "loc": "http://www.example.com/sitemap1.xml.gz",
+                    "lastmod": "2004-10-01T18:23:17+00:00",
+                },
+                {
+                    "loc": "http://www.example.com/sitemap2.xml.gz",
+                    "lastmod": "2005-01-01",
+                },
+            ],
         )
 
     def test_sitemap_strip(self):
         """Assert we can deal with trailing spaces inside <loc> tags - we've
         seen those
         """
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
   <url>
     <loc> http://www.example.com/</loc>
@@ -69,19 +88,26 @@ class SitemapTest(unittest.TestCase):
     <lastmod />
   </url>
 </urlset>
-""")
+"""
+        )
         self.assertEqual(
             list(s),
             [
-                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-                {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ]
+                {
+                    "priority": "1",
+                    "loc": "http://www.example.com/",
+                    "lastmod": "2009-08-16",
+                    "changefreq": "daily",
+                },
+                {"loc": "http://www.example.com/2", "lastmod": ""},
+            ],
         )
 
     def test_sitemap_wrong_ns(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
         with these, though is not 100% confirmed"""
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
   <url xmlns="">
     <loc> http://www.example.com/</loc>
@@ -94,19 +120,26 @@ class SitemapTest(unittest.TestCase):
     <lastmod />
   </url>
 </urlset>
-""")
+"""
+        )
         self.assertEqual(
             list(s),
             [
-                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-                {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ]
+                {
+                    "priority": "1",
+                    "loc": "http://www.example.com/",
+                    "lastmod": "2009-08-16",
+                    "changefreq": "daily",
+                },
+                {"loc": "http://www.example.com/2", "lastmod": ""},
+            ],
         )
 
     def test_sitemap_wrong_ns2(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
         with these, though is not 100% confirmed"""
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
 <urlset>
   <url xmlns="">
     <loc> http://www.example.com/</loc>
@@ -119,14 +152,20 @@ class SitemapTest(unittest.TestCase):
     <lastmod />
   </url>
 </urlset>
-""")
-        assert s.type == 'urlset'
+"""
+        )
+        assert s.type == "urlset"
         self.assertEqual(
             list(s),
             [
-                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-                {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ]
+                {
+                    "priority": "1",
+                    "loc": "http://www.example.com/",
+                    "lastmod": "2009-08-16",
+                    "changefreq": "daily",
+                },
+                {"loc": "http://www.example.com/2", "lastmod": ""},
+            ],
         )
 
     def test_sitemap_urls_from_robots(self):
@@ -148,15 +187,20 @@ Sitemap: /sitemap-relative-url.xml
 Disallow: /forum/search/
 Disallow: /forum/active/
 """
-        self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')),
-                         ['http://example.com/sitemap.xml',
-                          'http://example.com/sitemap-product-index.xml',
-                          'http://example.com/sitemap-uppercase.xml',
-                          'http://example.com/sitemap-relative-url.xml'])
+        self.assertEqual(
+            list(sitemap_urls_from_robots(robots, base_url="http://example.com")),
+            [
+                "http://example.com/sitemap.xml",
+                "http://example.com/sitemap-product-index.xml",
+                "http://example.com/sitemap-uppercase.xml",
+                "http://example.com/sitemap-relative-url.xml",
+            ],
+        )
 
     def test_sitemap_blanklines(self):
         """Assert we can deal with starting blank lines before <xml> tag"""
-        s = Sitemap(b"""
+        s = Sitemap(
+            b"""
 <?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 
@@ -178,29 +222,34 @@ Disallow: /forum/active/
 
 <!-- end cache -->
 </sitemapindex>
-""")
-        self.assertEqual(list(s), [
-            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'},
-            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'},
-            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'},
-        ])
+"""
+        )
+        self.assertEqual(
+            list(s),
+            [
+                {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},
+                {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},
+                {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
+            ],
+        )
 
     def test_comment(self):
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
     <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
         xmlns:xhtml="http://www.w3.org/1999/xhtml">
         <url>
             <loc>http://www.example.com/</loc>
             <!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->
         </url>
-    </urlset>""")
+    </urlset>"""
+        )
 
-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/'}
-        ])
+        self.assertEqual(list(s), [{"loc": "http://www.example.com/"}])
 
     def test_alternate(self):
-        s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="UTF-8"?>
     <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
         xmlns:xhtml="http://www.w3.org/1999/xhtml">
         <url>
@@ -213,24 +262,26 @@ Disallow: /forum/active/
                 href="http://www.example.com/english/"/>
             <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->
         </url>
-    </urlset>""")
+    </urlset>"""
+        )
 
         self.assertEqual(
             list(s),
             [
                 {
-                    'loc': 'http://www.example.com/english/',
-                    'alternate': [
-                        'http://www.example.com/deutsch/',
-                        'http://www.example.com/schweiz-deutsch/',
-                        'http://www.example.com/english/',
+                    "loc": "http://www.example.com/english/",
+                    "alternate": [
+                        "http://www.example.com/deutsch/",
+                        "http://www.example.com/schweiz-deutsch/",
+                        "http://www.example.com/english/",
                     ],
                 }
-            ]
+            ],
         )
 
     def test_xml_entity_expansion(self):
-        s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
+        s = Sitemap(
+            b"""<?xml version="1.0" encoding="utf-8"?>
           <!DOCTYPE foo [
           <!ELEMENT foo ANY >
           <!ENTITY xxe SYSTEM "file:///etc/passwd" >
@@ -240,10 +291,11 @@ Disallow: /forum/active/
               <loc>http://127.0.0.1:8000/&xxe;</loc>
             </url>
           </urlset>
-        """)
+        """
+        )
 
-        self.assertEqual(list(s), [{'loc': 'http://127.0.0.1:8000/'}])
+        self.assertEqual(list(s), [{"loc": "http://127.0.0.1:8000/"}])
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 3c87268ab..6fb7b8b82 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -7,18 +7,17 @@ from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
 
 class MySpider1(Spider):
-    name = 'myspider1'
+    name = "myspider1"
 
 
 class MySpider2(Spider):
-    name = 'myspider2'
+    name = "myspider2"
 
 
 class UtilsSpidersTestCase(unittest.TestCase):
-
     def test_iterate_spider_output(self):
         i = Item()
-        r = Request('http://scrapytest.org')
+        r = Request("http://scrapytest.org")
         o = object()
 
         self.assertEqual(list(iterate_spider_output(i)), [i])
@@ -28,6 +27,7 @@ class UtilsSpidersTestCase(unittest.TestCase):
 
     def test_iter_spider_classes(self):
         import tests.test_utils_spider
+
         it = iter_spider_classes(tests.test_utils_spider)
         self.assertEqual(set(it), {MySpider1, MySpider2})
 
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 4f1e8772b..45e23f793 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -5,11 +5,10 @@ import unittest
 from scrapy.utils.template import render_templatefile
 
 
-__doctests__ = ['scrapy.utils.template']
+__doctests__ = ["scrapy.utils.template"]
 
 
 class UtilsRenderTemplateFileTestCase(unittest.TestCase):
-
     def setUp(self):
         self.tmp_path = mkdtemp()
 
@@ -18,24 +17,24 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
 
     def test_simple_render(self):
 
-        context = dict(project_name='proj', name='spi', classname='TheSpider')
-        template = 'from ${project_name}.spiders.${name} import ${classname}'
-        rendered = 'from proj.spiders.spi import TheSpider'
+        context = dict(project_name="proj", name="spi", classname="TheSpider")
+        template = "from ${project_name}.spiders.${name} import ${classname}"
+        rendered = "from proj.spiders.spi import TheSpider"
 
-        template_path = Path(self.tmp_path, 'templ.py.tmpl')
-        render_path = Path(self.tmp_path, 'templ.py')
+        template_path = Path(self.tmp_path, "templ.py.tmpl")
+        render_path = Path(self.tmp_path, "templ.py")
 
-        template_path.write_text(template, encoding='utf8')
+        template_path.write_text(template, encoding="utf8")
         assert template_path.is_file()  # Failure of test itself
 
         render_templatefile(template_path, **context)
 
         self.assertFalse(template_path.exists())
-        self.assertEqual(render_path.read_text(encoding='utf8'), rendered)
+        self.assertEqual(render_path.read_text(encoding="utf8"), rendered)
 
         render_path.unlink()
         assert not render_path.exists()  # Failure of test itself
 
 
-if '__main__' == __name__:
+if "__main__" == __name__:
     unittest.main()
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index b8e8c3130..35d1508c6 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -17,7 +17,6 @@ class Bar(trackref.object_ref):
 
 
 class TrackrefTestCase(unittest.TestCase):
-
     def setUp(self):
         trackref.live_refs.clear()
 
@@ -27,34 +26,39 @@ class TrackrefTestCase(unittest.TestCase):
         o3 = Foo()  # NOQA
         self.assertEqual(
             trackref.format_live_refs(),
-            '''\
+            """\
 Live References
 
 Bar                                 1   oldest: 0s ago
 Foo                                 2   oldest: 0s ago
-''')
+""",
+        )
 
         self.assertEqual(
             trackref.format_live_refs(ignore=Foo),
-            '''\
+            """\
 Live References
 
 Bar                                 1   oldest: 0s ago
-''')
+""",
+        )
 
-    @mock.patch('sys.stdout', new_callable=StringIO)
+    @mock.patch("sys.stdout", new_callable=StringIO)
     def test_print_live_refs_empty(self, stdout):
         trackref.print_live_refs()
-        self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
+        self.assertEqual(stdout.getvalue(), "Live References\n\n\n")
 
-    @mock.patch('sys.stdout', new_callable=StringIO)
+    @mock.patch("sys.stdout", new_callable=StringIO)
     def test_print_live_refs_with_objects(self, stdout):
         o1 = Foo()  # NOQA
         trackref.print_live_refs()
-        self.assertEqual(stdout.getvalue(), '''\
+        self.assertEqual(
+            stdout.getvalue(),
+            """\
 Live References
 
-Foo                                 1   oldest: 0s ago\n\n''')
+Foo                                 1   oldest: 0s ago\n\n""",
+        )
 
     def test_get_oldest(self):
         o1 = Foo()  # NOQA
@@ -68,18 +72,18 @@ Foo                                 1   oldest: 0s ago\n\n''')
             sleep(0.01)
             o3_time = time()
         if o3_time <= o1_time:
-            raise SkipTest('time.time is not precise enough')
+            raise SkipTest("time.time is not precise enough")
 
         o3 = Foo()  # NOQA
-        self.assertIs(trackref.get_oldest('Foo'), o1)
-        self.assertIs(trackref.get_oldest('Bar'), o2)
-        self.assertIsNone(trackref.get_oldest('XXX'))
+        self.assertIs(trackref.get_oldest("Foo"), o1)
+        self.assertIs(trackref.get_oldest("Bar"), o2)
+        self.assertIsNone(trackref.get_oldest("XXX"))
 
     def test_iter_all(self):
         o1 = Foo()  # NOQA
         o2 = Bar()  # NOQA
         o3 = Foo()  # NOQA
         self.assertEqual(
-            set(trackref.iter_all('Foo')),
+            set(trackref.iter_all("Foo")),
             {o1, o3},
         )
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 58e2be622..9133663d9 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -13,207 +13,301 @@ from scrapy.utils.url import (
     url_has_any_extension,
 )
 
-__doctests__ = ['scrapy.utils.url']
+__doctests__ = ["scrapy.utils.url"]
 
 
 class UrlUtilsTest(unittest.TestCase):
-
     def test_url_is_from_any_domain(self):
-        url = 'http://www.wheele-bin-art.co.uk/get/product/123'
-        self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk']))
-        self.assertFalse(url_is_from_any_domain(url, ['art.co.uk']))
+        url = "http://www.wheele-bin-art.co.uk/get/product/123"
+        self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"]))
+        self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"]))
 
-        url = 'http://wheele-bin-art.co.uk/get/product/123'
-        self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk']))
-        self.assertFalse(url_is_from_any_domain(url, ['art.co.uk']))
+        url = "http://wheele-bin-art.co.uk/get/product/123"
+        self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"]))
+        self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"]))
 
-        url = 'http://www.Wheele-Bin-Art.co.uk/get/product/123'
-        self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.CO.UK']))
-        self.assertTrue(url_is_from_any_domain(url, ['WHEELE-BIN-ART.CO.UK']))
+        url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
+        self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"]))
+        self.assertTrue(url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"]))
 
-        url = 'http://192.169.0.15:8080/mypage.html'
-        self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080']))
-        self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15']))
+        url = "http://192.169.0.15:8080/mypage.html"
+        self.assertTrue(url_is_from_any_domain(url, ["192.169.0.15:8080"]))
+        self.assertFalse(url_is_from_any_domain(url, ["192.169.0.15"]))
 
         url = (
-            'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20'
-            'javascript:%20document.orderform_2581_1190810811.submit%28%29'
+            "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
+            "javascript:%20document.orderform_2581_1190810811.submit%28%29"
+        )
+        self.assertFalse(url_is_from_any_domain(url, ["testdomain.com"]))
+        self.assertFalse(
+            url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
         )
-        self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
-        self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))
 
     def test_url_is_from_spider(self):
-        spider = Spider(name='example.com')
-        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
-        self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider))
-        self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider))
-        self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', spider))
+        spider = Spider(name="example.com")
+        self.assertTrue(
+            url_is_from_spider("http://www.example.com/some/page.html", spider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://sub.example.com/some/page.html", spider)
+        )
+        self.assertFalse(
+            url_is_from_spider("http://www.example.org/some/page.html", spider)
+        )
+        self.assertFalse(
+            url_is_from_spider("http://www.example.net/some/page.html", spider)
+        )
 
     def test_url_is_from_spider_class_attributes(self):
         class MySpider(Spider):
-            name = 'example.com'
-        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider))
-        self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider))
-        self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', MySpider))
-        self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
+            name = "example.com"
+
+        self.assertTrue(
+            url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+        )
+        self.assertFalse(
+            url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+        )
+        self.assertFalse(
+            url_is_from_spider("http://www.example.net/some/page.html", MySpider)
+        )
 
     def test_url_is_from_spider_with_allowed_domains(self):
-        spider = Spider(name='example.com', allowed_domains=['example.org', 'example.net'])
-        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
-        self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider))
-        self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider))
-        self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider))
-        self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
-        self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
+        spider = Spider(
+            name="example.com", allowed_domains=["example.org", "example.net"]
+        )
+        self.assertTrue(
+            url_is_from_spider("http://www.example.com/some/page.html", spider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://sub.example.com/some/page.html", spider)
+        )
+        self.assertTrue(url_is_from_spider("http://example.com/some/page.html", spider))
+        self.assertTrue(
+            url_is_from_spider("http://www.example.org/some/page.html", spider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://www.example.net/some/page.html", spider)
+        )
+        self.assertFalse(
+            url_is_from_spider("http://www.example.us/some/page.html", spider)
+        )
 
-        spider = Spider(name='example.com', allowed_domains={'example.com', 'example.net'})
-        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
+        spider = Spider(
+            name="example.com", allowed_domains={"example.com", "example.net"}
+        )
+        self.assertTrue(
+            url_is_from_spider("http://www.example.com/some/page.html", spider)
+        )
 
-        spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net'))
-        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
+        spider = Spider(
+            name="example.com", allowed_domains=("example.com", "example.net")
+        )
+        self.assertTrue(
+            url_is_from_spider("http://www.example.com/some/page.html", spider)
+        )
 
     def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
         class MySpider(Spider):
-            name = 'example.com'
-            allowed_domains = ('example.org', 'example.net')
-        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider))
-        self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider))
-        self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider))
-        self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySpider))
-        self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
-        self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
+            name = "example.com"
+            allowed_domains = ("example.org", "example.net")
+
+        self.assertTrue(
+            url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://example.com/some/page.html", MySpider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+        )
+        self.assertTrue(
+            url_is_from_spider("http://www.example.net/some/page.html", MySpider)
+        )
+        self.assertFalse(
+            url_is_from_spider("http://www.example.us/some/page.html", MySpider)
+        )
 
     def test_url_has_any_extension(self):
-        deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
-        self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions))
-        self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions))
-        self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions))
-        self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions))
-        self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions))
-        self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions))
+        deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
+        self.assertTrue(
+            url_has_any_extension(
+                "http://www.example.com/archive.tar.gz", deny_extensions
+            )
+        )
+        self.assertTrue(
+            url_has_any_extension("http://www.example.com/page.doc", deny_extensions)
+        )
+        self.assertTrue(
+            url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)
+        )
+        self.assertFalse(
+            url_has_any_extension("http://www.example.com/page.htm", deny_extensions)
+        )
+        self.assertFalse(
+            url_has_any_extension("http://www.example.com/", deny_extensions)
+        )
+        self.assertFalse(
+            url_has_any_extension(
+                "http://www.example.com/page.doc.html", deny_extensions
+            )
+        )
 
 
 class AddHttpIfNoScheme(unittest.TestCase):
-
     def test_add_scheme(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme("www.example.com"), "http://www.example.com"
+        )
 
     def test_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
+        self.assertEqual(add_http_if_no_scheme("example.com"), "http://example.com")
 
     def test_path(self):
         self.assertEqual(
-            add_http_if_no_scheme('www.example.com/some/page.html'),
-            'http://www.example.com/some/page.html')
+            add_http_if_no_scheme("www.example.com/some/page.html"),
+            "http://www.example.com/some/page.html",
+        )
 
     def test_port(self):
         self.assertEqual(
-            add_http_if_no_scheme('www.example.com:80'),
-            'http://www.example.com:80')
+            add_http_if_no_scheme("www.example.com:80"), "http://www.example.com:80"
+        )
 
     def test_fragment(self):
         self.assertEqual(
-            add_http_if_no_scheme('www.example.com/some/page#frag'),
-            'http://www.example.com/some/page#frag')
+            add_http_if_no_scheme("www.example.com/some/page#frag"),
+            "http://www.example.com/some/page#frag",
+        )
 
     def test_query(self):
         self.assertEqual(
-            add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
-            'http://www.example.com/do?a=1&b=2&c=3')
+            add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3"),
+            "http://www.example.com/do?a=1&b=2&c=3",
+        )
 
     def test_username_password(self):
         self.assertEqual(
-            add_http_if_no_scheme('username:password@www.example.com'),
-            'http://username:password@www.example.com')
+            add_http_if_no_scheme("username:password@www.example.com"),
+            "http://username:password@www.example.com",
+        )
 
     def test_complete_url(self):
         self.assertEqual(
-            add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+            add_http_if_no_scheme(
+                "username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
+            ),
+            "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+        )
 
     def test_preserve_http(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme("http://www.example.com"), "http://www.example.com"
+        )
 
     def test_preserve_http_without_subdomain(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://example.com'),
-            'http://example.com')
+            add_http_if_no_scheme("http://example.com"), "http://example.com"
+        )
 
     def test_preserve_http_path(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://www.example.com/some/page.html'),
-            'http://www.example.com/some/page.html')
+            add_http_if_no_scheme("http://www.example.com/some/page.html"),
+            "http://www.example.com/some/page.html",
+        )
 
     def test_preserve_http_port(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://www.example.com:80'),
-            'http://www.example.com:80')
+            add_http_if_no_scheme("http://www.example.com:80"),
+            "http://www.example.com:80",
+        )
 
     def test_preserve_http_fragment(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://www.example.com/some/page#frag'),
-            'http://www.example.com/some/page#frag')
+            add_http_if_no_scheme("http://www.example.com/some/page#frag"),
+            "http://www.example.com/some/page#frag",
+        )
 
     def test_preserve_http_query(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
-            'http://www.example.com/do?a=1&b=2&c=3')
+            add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3"),
+            "http://www.example.com/do?a=1&b=2&c=3",
+        )
 
     def test_preserve_http_username_password(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://username:password@www.example.com'),
-            'http://username:password@www.example.com')
+            add_http_if_no_scheme("http://username:password@www.example.com"),
+            "http://username:password@www.example.com",
+        )
 
     def test_preserve_http_complete_url(self):
         self.assertEqual(
-            add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+            add_http_if_no_scheme(
+                "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
+            ),
+            "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+        )
 
     def test_protocol_relative(self):
         self.assertEqual(
-            add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
+            add_http_if_no_scheme("//www.example.com"), "http://www.example.com"
+        )
 
     def test_protocol_relative_without_subdomain(self):
-        self.assertEqual(
-            add_http_if_no_scheme('//example.com'), 'http://example.com')
+        self.assertEqual(add_http_if_no_scheme("//example.com"), "http://example.com")
 
     def test_protocol_relative_path(self):
         self.assertEqual(
-            add_http_if_no_scheme('//www.example.com/some/page.html'),
-            'http://www.example.com/some/page.html')
+            add_http_if_no_scheme("//www.example.com/some/page.html"),
+            "http://www.example.com/some/page.html",
+        )
 
     def test_protocol_relative_port(self):
         self.assertEqual(
-            add_http_if_no_scheme('//www.example.com:80'),
-            'http://www.example.com:80')
+            add_http_if_no_scheme("//www.example.com:80"), "http://www.example.com:80"
+        )
 
     def test_protocol_relative_fragment(self):
         self.assertEqual(
-            add_http_if_no_scheme('//www.example.com/some/page#frag'),
-            'http://www.example.com/some/page#frag')
+            add_http_if_no_scheme("//www.example.com/some/page#frag"),
+            "http://www.example.com/some/page#frag",
+        )
 
     def test_protocol_relative_query(self):
         self.assertEqual(
-            add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
-            'http://www.example.com/do?a=1&b=2&c=3')
+            add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3"),
+            "http://www.example.com/do?a=1&b=2&c=3",
+        )
 
     def test_protocol_relative_username_password(self):
         self.assertEqual(
-            add_http_if_no_scheme('//username:password@www.example.com'),
-            'http://username:password@www.example.com')
+            add_http_if_no_scheme("//username:password@www.example.com"),
+            "http://username:password@www.example.com",
+        )
 
     def test_protocol_relative_complete_url(self):
         self.assertEqual(
-            add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+            add_http_if_no_scheme(
+                "//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
+            ),
+            "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+        )
 
     def test_preserve_https(self):
         self.assertEqual(
-            add_http_if_no_scheme('https://www.example.com'),
-            'https://www.example.com')
+            add_http_if_no_scheme("https://www.example.com"), "https://www.example.com"
+        )
 
     def test_preserve_ftp(self):
-        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme("ftp://www.example.com"), "ftp://www.example.com"
+        )
 
 
 class GuessSchemeTest(unittest.TestCase):
@@ -223,8 +317,10 @@ class GuessSchemeTest(unittest.TestCase):
 def create_guess_scheme_t(args):
     def do_expected(self):
         url = guess_scheme(args[0])
-        assert url.startswith(args[1]), \
-            f'Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`'
+        assert url.startswith(
+            args[1]
+        ), f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`"
+
     return do_expected
 
 
@@ -233,100 +329,100 @@ def create_skipped_scheme_t(args):
         raise unittest.SkipTest(args[2])
         url = guess_scheme(args[0])
         assert url.startswith(args[1])
+
     return do_expected
 
 
 for k, args in enumerate(
     [
-        ('/index', 'file://'),
-        ('/index.html', 'file://'),
-        ('./index.html', 'file://'),
-        ('../index.html', 'file://'),
-        ('../../index.html', 'file://'),
-        ('./data/index.html', 'file://'),
-        ('.hidden/data/index.html', 'file://'),
-        ('/home/user/www/index.html', 'file://'),
-        ('//home/user/www/index.html', 'file://'),
-        ('file:///home/user/www/index.html', 'file://'),
-
-        ('index.html', 'http://'),
-        ('example.com', 'http://'),
-        ('www.example.com', 'http://'),
-        ('www.example.com/index.html', 'http://'),
-        ('http://example.com', 'http://'),
-        ('http://example.com/index.html', 'http://'),
-        ('localhost', 'http://'),
-        ('localhost/index.html', 'http://'),
-
+        ("/index", "file://"),
+        ("/index.html", "file://"),
+        ("./index.html", "file://"),
+        ("../index.html", "file://"),
+        ("../../index.html", "file://"),
+        ("./data/index.html", "file://"),
+        (".hidden/data/index.html", "file://"),
+        ("/home/user/www/index.html", "file://"),
+        ("//home/user/www/index.html", "file://"),
+        ("file:///home/user/www/index.html", "file://"),
+        ("index.html", "http://"),
+        ("example.com", "http://"),
+        ("www.example.com", "http://"),
+        ("www.example.com/index.html", "http://"),
+        ("http://example.com", "http://"),
+        ("http://example.com/index.html", "http://"),
+        ("localhost", "http://"),
+        ("localhost/index.html", "http://"),
         # some corner cases (default to http://)
-        ('/', 'http://'),
-        ('.../test', 'http://'),
+        ("/", "http://"),
+        (".../test", "http://"),
     ],
     start=1,
 ):
     t_method = create_guess_scheme_t(args)
-    t_method.__name__ = f'test_uri_{k:03}'
+    t_method.__name__ = f"test_uri_{k:03}"
     setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
 for k, args in enumerate(
     [
         (
-            r'C:\absolute\path\to\a\file.html',
-            'file://',
-            'Windows filepath are not supported for scrapy shell',
+            r"C:\absolute\path\to\a\file.html",
+            "file://",
+            "Windows filepath are not supported for scrapy shell",
         ),
     ],
     start=1,
 ):
     t_method = create_skipped_scheme_t(args)
-    t_method.__name__ = f'test_uri_skipped_{k:03}'
+    t_method.__name__ = f"test_uri_skipped_{k:03}"
     setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 
 class StripUrl(unittest.TestCase):
-
     def test_noop(self):
-        self.assertEqual(strip_url(
-            'http://www.example.com/index.html'),
-            'http://www.example.com/index.html')
+        self.assertEqual(
+            strip_url("http://www.example.com/index.html"),
+            "http://www.example.com/index.html",
+        )
 
     def test_noop_query_string(self):
-        self.assertEqual(strip_url(
-            'http://www.example.com/index.html?somekey=somevalue'),
-            'http://www.example.com/index.html?somekey=somevalue')
+        self.assertEqual(
+            strip_url("http://www.example.com/index.html?somekey=somevalue"),
+            "http://www.example.com/index.html?somekey=somevalue",
+        )
 
     def test_fragments(self):
-        self.assertEqual(strip_url(
-            'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),
-            'http://www.example.com/index.html?somekey=somevalue#section')
+        self.assertEqual(
+            strip_url(
+                "http://www.example.com/index.html?somekey=somevalue#section",
+                strip_fragment=False,
+            ),
+            "http://www.example.com/index.html?somekey=somevalue#section",
+        )
 
     def test_path(self):
         for input_url, origin, output_url in [
-            ('http://www.example.com/',
-             False,
-             'http://www.example.com/'),
-
-            ('http://www.example.com',
-             False,
-             'http://www.example.com'),
-
-            ('http://www.example.com',
-             True,
-             'http://www.example.com/'),
+            ("http://www.example.com/", False, "http://www.example.com/"),
+            ("http://www.example.com", False, "http://www.example.com"),
+            ("http://www.example.com", True, "http://www.example.com/"),
         ]:
             self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
 
     def test_credentials(self):
         for i, o in [
-            ('http://username@www.example.com/index.html?somekey=somevalue#section',
-             'http://www.example.com/index.html?somekey=somevalue'),
-
-            ('https://username:@www.example.com/index.html?somekey=somevalue#section',
-             'https://www.example.com/index.html?somekey=somevalue'),
-
-            ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
-             'ftp://www.example.com/index.html?somekey=somevalue'),
+            (
+                "http://username@www.example.com/index.html?somekey=somevalue#section",
+                "http://www.example.com/index.html?somekey=somevalue",
+            ),
+            (
+                "https://username:@www.example.com/index.html?somekey=somevalue#section",
+                "https://www.example.com/index.html?somekey=somevalue",
+            ),
+            (
+                "ftp://username:password@www.example.com/index.html?somekey=somevalue#section",
+                "ftp://www.example.com/index.html?somekey=somevalue",
+            ),
         ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
@@ -334,124 +430,163 @@ class StripUrl(unittest.TestCase):
         for i, o in [
             # user: "username@"
             # password: none
-            ('http://username%40@www.example.com/index.html?somekey=somevalue#section',
-             'http://www.example.com/index.html?somekey=somevalue'),
-
+            (
+                "http://username%40@www.example.com/index.html?somekey=somevalue#section",
+                "http://www.example.com/index.html?somekey=somevalue",
+            ),
             # user: "username:pass"
             # password: ""
-            ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section',
-             'https://www.example.com/index.html?somekey=somevalue'),
-
+            (
+                "https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
+                "https://www.example.com/index.html?somekey=somevalue",
+            ),
             # user: "me"
             # password: "user@domain.com"
-            ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
-             'ftp://www.example.com/index.html?somekey=somevalue'),
+            (
+                "ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section",
+                "ftp://www.example.com/index.html?somekey=somevalue",
+            ),
         ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_default_ports_creds_off(self):
         for i, o in [
-            ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section',
-             'http://www.example.com/index.html?somekey=somevalue'),
-
-            ('http://username:password@www.example.com:8080/index.html#section',
-             'http://www.example.com:8080/index.html'),
-
-            ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section',
-             'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),
-
-            ('https://username:password@www.example.com:443/index.html',
-             'https://www.example.com/index.html'),
-
-            ('https://username:password@www.example.com:442/index.html',
-             'https://www.example.com:442/index.html'),
-
-            ('https://username:password@www.example.com:80/index.html',
-             'https://www.example.com:80/index.html'),
-
-            ('ftp://username:password@www.example.com:21/file.txt',
-             'ftp://www.example.com/file.txt'),
-
-            ('ftp://username:password@www.example.com:221/file.txt',
-             'ftp://www.example.com:221/file.txt'),
+            (
+                "http://username:password@www.example.com:80/index.html?somekey=somevalue#section",
+                "http://www.example.com/index.html?somekey=somevalue",
+            ),
+            (
+                "http://username:password@www.example.com:8080/index.html#section",
+                "http://www.example.com:8080/index.html",
+            ),
+            (
+                "http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section",
+                "http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov",
+            ),
+            (
+                "https://username:password@www.example.com:443/index.html",
+                "https://www.example.com/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:442/index.html",
+                "https://www.example.com:442/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:80/index.html",
+                "https://www.example.com:80/index.html",
+            ),
+            (
+                "ftp://username:password@www.example.com:21/file.txt",
+                "ftp://www.example.com/file.txt",
+            ),
+            (
+                "ftp://username:password@www.example.com:221/file.txt",
+                "ftp://www.example.com:221/file.txt",
+            ),
         ]:
             self.assertEqual(strip_url(i), o)
 
     def test_default_ports(self):
         for i, o in [
-            ('http://username:password@www.example.com:80/index.html',
-             'http://username:password@www.example.com/index.html'),
-
-            ('http://username:password@www.example.com:8080/index.html',
-             'http://username:password@www.example.com:8080/index.html'),
-
-            ('http://username:password@www.example.com:443/index.html',
-             'http://username:password@www.example.com:443/index.html'),
-
-            ('https://username:password@www.example.com:443/index.html',
-             'https://username:password@www.example.com/index.html'),
-
-            ('https://username:password@www.example.com:442/index.html',
-             'https://username:password@www.example.com:442/index.html'),
-
-            ('https://username:password@www.example.com:80/index.html',
-             'https://username:password@www.example.com:80/index.html'),
-
-            ('ftp://username:password@www.example.com:21/file.txt',
-             'ftp://username:password@www.example.com/file.txt'),
-
-            ('ftp://username:password@www.example.com:221/file.txt',
-             'ftp://username:password@www.example.com:221/file.txt'),
+            (
+                "http://username:password@www.example.com:80/index.html",
+                "http://username:password@www.example.com/index.html",
+            ),
+            (
+                "http://username:password@www.example.com:8080/index.html",
+                "http://username:password@www.example.com:8080/index.html",
+            ),
+            (
+                "http://username:password@www.example.com:443/index.html",
+                "http://username:password@www.example.com:443/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:443/index.html",
+                "https://username:password@www.example.com/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:442/index.html",
+                "https://username:password@www.example.com:442/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:80/index.html",
+                "https://username:password@www.example.com:80/index.html",
+            ),
+            (
+                "ftp://username:password@www.example.com:21/file.txt",
+                "ftp://username:password@www.example.com/file.txt",
+            ),
+            (
+                "ftp://username:password@www.example.com:221/file.txt",
+                "ftp://username:password@www.example.com:221/file.txt",
+            ),
         ]:
-            self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
+            self.assertEqual(
+                strip_url(i, strip_default_port=True, strip_credentials=False), o
+            )
 
     def test_default_ports_keep(self):
         for i, o in [
-            ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section',
-             'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'),
-
-            ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section',
-             'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'),
-
-            ('http://username:password@www.example.com:443/index.html',
-             'http://username:password@www.example.com:443/index.html'),
-
-            ('https://username:password@www.example.com:443/index.html',
-             'https://username:password@www.example.com:443/index.html'),
-
-            ('https://username:password@www.example.com:442/index.html',
-             'https://username:password@www.example.com:442/index.html'),
-
-            ('https://username:password@www.example.com:80/index.html',
-             'https://username:password@www.example.com:80/index.html'),
-
-            ('ftp://username:password@www.example.com:21/file.txt',
-             'ftp://username:password@www.example.com:21/file.txt'),
-
-            ('ftp://username:password@www.example.com:221/file.txt',
-             'ftp://username:password@www.example.com:221/file.txt'),
+            (
+                "http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section",
+                "http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov",
+            ),
+            (
+                "http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section",
+                "http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov",
+            ),
+            (
+                "http://username:password@www.example.com:443/index.html",
+                "http://username:password@www.example.com:443/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:443/index.html",
+                "https://username:password@www.example.com:443/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:442/index.html",
+                "https://username:password@www.example.com:442/index.html",
+            ),
+            (
+                "https://username:password@www.example.com:80/index.html",
+                "https://username:password@www.example.com:80/index.html",
+            ),
+            (
+                "ftp://username:password@www.example.com:21/file.txt",
+                "ftp://username:password@www.example.com:21/file.txt",
+            ),
+            (
+                "ftp://username:password@www.example.com:221/file.txt",
+                "ftp://username:password@www.example.com:221/file.txt",
+            ),
         ]:
-            self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
+            self.assertEqual(
+                strip_url(i, strip_default_port=False, strip_credentials=False), o
+            )
 
     def test_origin_only(self):
         for i, o in [
-            ('http://username:password@www.example.com/index.html',
-             'http://www.example.com/'),
-
-            ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag',
-             'http://www.example.com/'),
-
-            ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag',
-             'http://www.example.com:8008/'),
-
-            ('https://username:password@www.example.com:443/index.html',
-             'https://www.example.com/'),
+            (
+                "http://username:password@www.example.com/index.html",
+                "http://www.example.com/",
+            ),
+            (
+                "http://username:password@www.example.com:80/foo/bar?query=value#somefrag",
+                "http://www.example.com/",
+            ),
+            (
+                "http://username:password@www.example.com:8008/foo/bar?query=value#somefrag",
+                "http://www.example.com:8008/",
+            ),
+            (
+                "https://username:password@www.example.com:443/index.html",
+                "https://www.example.com/",
+            ),
         ]:
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 
 class IsPathTestCase(unittest.TestCase):
-
     def test_path(self):
         for input_value, output_value in (
             # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
@@ -467,10 +602,11 @@ class IsPathTestCase(unittest.TestCase):
             (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
             (r"\\?\C:\user\docs\Letter.txt", True),
             (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
-
             (r"https://example.com", False),
         ):
-            self.assertEqual(_is_filesystem_path(input_value), output_value, input_value)
+            self.assertEqual(
+                _is_filesystem_path(input_value), output_value, input_value
+            )
 
 
 if __name__ == "__main__":
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 3dcaf71dd..aadfe0f40 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -9,6 +9,7 @@ import OpenSSL.SSL
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
 from twisted.internet import reactor, defer
+
 try:
     from twisted.internet.testing import StringTransport
 except ImportError:
@@ -37,17 +38,24 @@ from tests.mockserver import (
 
 def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
     """Adapted version of twisted.web.client.getPage"""
+
     def _clientfactory(url, *args, **kwargs):
         url = to_unicode(url)
-        timeout = kwargs.pop('timeout', 0)
+        timeout = kwargs.pop("timeout", 0)
         f = client.ScrapyHTTPClientFactory(
-            Request(url, *args, **kwargs), timeout=timeout)
+            Request(url, *args, **kwargs), timeout=timeout
+        )
         f.deferred.addCallback(response_transform or (lambda r: r.body))
         return f
 
     from twisted.web.client import _makeGetterFactory
+
     return _makeGetterFactory(
-        to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs,
+        to_bytes(url),
+        _clientfactory,
+        contextFactory=contextFactory,
+        *args,
+        **kwargs,
     ).deferred
 
 
@@ -59,51 +67,80 @@ class ParseUrlTestCase(unittest.TestCase):
         return (f.scheme, f.netloc, f.host, f.port, f.path)
 
     def testParse(self):
-        lip = '127.0.0.1'
+        lip = "127.0.0.1"
         tests = (
-            ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-            ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-            ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
-            ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-            ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-            ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
-
-            ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
-            ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
-            ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
-            ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
-            ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-            ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
-            ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
-            ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
-
-            ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
-            ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-            ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
-
-            ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
-            ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
+            (
+                "http://127.0.0.1?c=v&c2=v2#fragment",
+                ("http", lip, lip, 80, "/?c=v&c2=v2"),
+            ),
+            (
+                "http://127.0.0.1/?c=v&c2=v2#fragment",
+                ("http", lip, lip, 80, "/?c=v&c2=v2"),
+            ),
+            (
+                "http://127.0.0.1/foo?c=v&c2=v2#frag",
+                ("http", lip, lip, 80, "/foo?c=v&c2=v2"),
+            ),
+            (
+                "http://127.0.0.1:100?c=v&c2=v2#fragment",
+                ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"),
+            ),
+            (
+                "http://127.0.0.1:100/?c=v&c2=v2#frag",
+                ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"),
+            ),
+            (
+                "http://127.0.0.1:100/foo?c=v&c2=v2#frag",
+                ("http", lip + ":100", lip, 100, "/foo?c=v&c2=v2"),
+            ),
+            ("http://127.0.0.1", ("http", lip, lip, 80, "/")),
+            ("http://127.0.0.1/", ("http", lip, lip, 80, "/")),
+            ("http://127.0.0.1/foo", ("http", lip, lip, 80, "/foo")),
+            ("http://127.0.0.1?param=value", ("http", lip, lip, 80, "/?param=value")),
+            ("http://127.0.0.1/?param=value", ("http", lip, lip, 80, "/?param=value")),
+            (
+                "http://127.0.0.1:12345/foo",
+                ("http", lip + ":12345", lip, 12345, "/foo"),
+            ),
+            ("http://spam:12345/foo", ("http", "spam:12345", "spam", 12345, "/foo")),
+            (
+                "http://spam.test.org/foo",
+                ("http", "spam.test.org", "spam.test.org", 80, "/foo"),
+            ),
+            ("https://127.0.0.1/foo", ("https", lip, lip, 443, "/foo")),
+            (
+                "https://127.0.0.1/?param=value",
+                ("https", lip, lip, 443, "/?param=value"),
+            ),
+            ("https://127.0.0.1:12345/", ("https", lip + ":12345", lip, 12345, "/")),
+            (
+                "http://scrapytest.org/foo ",
+                ("http", "scrapytest.org", "scrapytest.org", 80, "/foo"),
+            ),
+            ("http://egg:7890 ", ("http", "egg:7890", "egg", 7890, "/")),
         )
 
         for url, test in tests:
-            test = tuple(
-                to_bytes(x) if not isinstance(x, int) else x for x in test)
+            test = tuple(to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
-
     def test_earlyHeaders(self):
         # basic test stolen from twisted HTTPageGetter
-        factory = client.ScrapyHTTPClientFactory(Request(
-            url='http://foo/bar',
-            body="some data",
-            headers={
-                'Host': 'example.net',
-                'User-Agent': 'fooble',
-                'Cookie': 'blah blah',
-                'Content-Length': '12981',
-                'Useful': 'value'}))
+        factory = client.ScrapyHTTPClientFactory(
+            Request(
+                url="http://foo/bar",
+                body="some data",
+                headers={
+                    "Host": "example.net",
+                    "User-Agent": "fooble",
+                    "Cookie": "blah blah",
+                    "Content-Length": "12981",
+                    "Useful": "value",
+                },
+            )
+        )
 
         self._test(
             factory,
@@ -115,22 +152,22 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             b"Host: example.net\r\n"
             b"Cookie: blah blah\r\n"
             b"\r\n"
-            b"some data")
+            b"some data",
+        )
 
         # test minimal sent headers
-        factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
-        self._test(
-            factory,
-            b"GET /bar HTTP/1.0\r\n"
-            b"Host: foo\r\n"
-            b"\r\n")
+        factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar"))
+        self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n")
 
         # test a simple POST with body and content-type
-        factory = client.ScrapyHTTPClientFactory(Request(
-            method='POST',
-            url='http://foo/bar',
-            body='name=value',
-            headers={'Content-Type': 'application/x-www-form-urlencoded'}))
+        factory = client.ScrapyHTTPClientFactory(
+            Request(
+                method="POST",
+                url="http://foo/bar",
+                body="name=value",
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+            )
+        )
 
         self._test(
             factory,
@@ -140,29 +177,29 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             b"Content-Type: application/x-www-form-urlencoded\r\n"
             b"Content-Length: 10\r\n"
             b"\r\n"
-            b"name=value")
+            b"name=value",
+        )
 
         # test a POST method with no body provided
-        factory = client.ScrapyHTTPClientFactory(Request(
-            method='POST',
-            url='http://foo/bar'
-        ))
+        factory = client.ScrapyHTTPClientFactory(
+            Request(method="POST", url="http://foo/bar")
+        )
 
         self._test(
             factory,
-            b"POST /bar HTTP/1.0\r\n"
-            b"Host: foo\r\n"
-            b"Content-Length: 0\r\n"
-            b"\r\n")
+            b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n",
+        )
 
         # test with single and multivalued headers
-        factory = client.ScrapyHTTPClientFactory(Request(
-            url='http://foo/bar',
-            headers={
-                'X-Meta-Single': 'single',
-                'X-Meta-Multivalued': ['value1', 'value2'],
-            },
-        ))
+        factory = client.ScrapyHTTPClientFactory(
+            Request(
+                url="http://foo/bar",
+                headers={
+                    "X-Meta-Single": "single",
+                    "X-Meta-Multivalued": ["value1", "value2"],
+                },
+            )
+        )
 
         self._test(
             factory,
@@ -171,16 +208,21 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             b"X-Meta-Multivalued: value1\r\n"
             b"X-Meta-Multivalued: value2\r\n"
             b"X-Meta-Single: single\r\n"
-            b"\r\n")
+            b"\r\n",
+        )
 
         # same test with single and multivalued headers but using Headers class
-        factory = client.ScrapyHTTPClientFactory(Request(
-            url='http://foo/bar',
-            headers=Headers({
-                'X-Meta-Single': 'single',
-                'X-Meta-Multivalued': ['value1', 'value2'],
-            }),
-        ))
+        factory = client.ScrapyHTTPClientFactory(
+            Request(
+                url="http://foo/bar",
+                headers=Headers(
+                    {
+                        "X-Meta-Single": "single",
+                        "X-Meta-Multivalued": ["value1", "value2"],
+                    }
+                ),
+            )
+        )
 
         self._test(
             factory,
@@ -189,7 +231,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             b"X-Meta-Multivalued: value1\r\n"
             b"X-Meta-Multivalued: value2\r\n"
             b"X-Meta-Single: single\r\n"
-            b"\r\n")
+            b"\r\n",
+        )
 
     def _test(self, factory, testvalue):
         transport = StringTransport()
@@ -197,14 +240,13 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         protocol.factory = factory
         protocol.makeConnection(transport)
         self.assertEqual(
-            set(transport.value().splitlines()),
-            set(testvalue.splitlines()))
+            set(transport.value().splitlines()), set(testvalue.splitlines())
+        )
         return testvalue
 
     def test_non_standard_line_endings(self):
         # regression test for: http://dev.scrapy.org/ticket/258
-        factory = client.ScrapyHTTPClientFactory(Request(
-            url='http://foo/bar'))
+        factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar"))
         protocol = client.ScrapyHTTPPageGetter()
         protocol.factory = factory
         protocol.headers = Headers()
@@ -212,15 +254,17 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         protocol.dataReceived(b"Hello: World\n")
         protocol.dataReceived(b"Foo: Bar\n")
         protocol.dataReceived(b"\n")
-        self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']}))
+        self.assertEqual(
+            protocol.headers, Headers({"Hello": ["World"], "Foo": ["Bar"]})
+        )
 
 
 class EncodingResource(resource.Resource):
-    out_encoding = 'cp1251'
+    out_encoding = "cp1251"
 
     def render(self, request):
         body = to_unicode(request.content.read())
-        request.setHeader(b'content-encoding', self.out_encoding)
+        request.setHeader(b"content-encoding", self.out_encoding)
         return body.encode(self.out_encoding)
 
 
@@ -257,16 +301,22 @@ class WebClientTestCase(unittest.TestCase):
     def testPayload(self):
         s = "0123456789" * 10
         return getPage(self.getURL("payload"), body=s).addCallback(
-            self.assertEqual, to_bytes(s))
+            self.assertEqual, to_bytes(s)
+        )
 
     def testHostHeader(self):
         # if we pass Host header explicitly, it should be used, otherwise
         # it should extract from url
-        return defer.gatherResults([
-            getPage(self.getURL("host")).addCallback(
-                self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")),
-            getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
-                self.assertEqual, to_bytes("www.example.com"))])
+        return defer.gatherResults(
+            [
+                getPage(self.getURL("host")).addCallback(
+                    self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")
+                ),
+                getPage(
+                    self.getURL("host"), headers={"Host": "www.example.com"}
+                ).addCallback(self.assertEqual, to_bytes("www.example.com")),
+            ]
+        )
 
     def test_getPage(self):
         """
@@ -283,11 +333,16 @@ class WebClientTestCase(unittest.TestCase):
         the empty string if the method is C{HEAD} and there is a successful
         response code.
         """
+
         def _getPage(method):
             return getPage(self.getURL("file"), method=method)
-        return defer.gatherResults([
-            _getPage("head").addCallback(self.assertEqual, b""),
-            _getPage("HEAD").addCallback(self.assertEqual, b"")])
+
+        return defer.gatherResults(
+            [
+                _getPage("head").addCallback(self.assertEqual, b""),
+                _getPage("HEAD").addCallback(self.assertEqual, b""),
+            ]
+        )
 
     def test_timeoutNotTriggering(self):
         """
@@ -296,8 +351,7 @@ class WebClientTestCase(unittest.TestCase):
         called back with the contents of the page.
         """
         d = getPage(self.getURL("host"), timeout=100)
-        d.addCallback(
-            self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}"))
+        d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}"))
         return d
 
     def test_timeoutTriggering(self):
@@ -307,8 +361,8 @@ class WebClientTestCase(unittest.TestCase):
         L{Deferred} is errbacked with a L{error.TimeoutError}.
         """
         finished = self.assertFailure(
-            getPage(self.getURL("wait"), timeout=0.000001),
-            defer.TimeoutError)
+            getPage(self.getURL("wait"), timeout=0.000001), defer.TimeoutError
+        )
 
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
@@ -319,27 +373,28 @@ class WebClientTestCase(unittest.TestCase):
             if connected:
                 connected[0].transport.loseConnection()
             return passthrough
+
         finished.addBoth(cleanup)
         return finished
 
     def testNotFound(self):
-        return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
+        return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile)
 
     def _cbNoSuchFile(self, pageData):
-        self.assertIn(b'404 - No Such Resource', pageData)
+        self.assertIn(b"404 - No Such Resource", pageData)
 
     def testFactoryInfo(self):
-        url = self.getURL('file')
+        url = self.getURL("file")
         _, _, host, port, _ = client._parse(url)
         factory = client.ScrapyHTTPClientFactory(Request(url))
         reactor.connectTCP(to_unicode(host), port, factory)
         return factory.deferred.addCallback(self._cbFactoryInfo, factory)
 
     def _cbFactoryInfo(self, ignoredResult, factory):
-        self.assertEqual(factory.status, b'200')
-        self.assertTrue(factory.version.startswith(b'HTTP/'))
-        self.assertEqual(factory.message, b'OK')
-        self.assertEqual(factory.response_headers[b'content-length'], b'10')
+        self.assertEqual(factory.status, b"200")
+        self.assertTrue(factory.version.startswith(b"HTTP/"))
+        self.assertEqual(factory.message, b"OK")
+        self.assertEqual(factory.response_headers[b"content-length"], b"10")
 
     def testRedirect(self):
         return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
@@ -349,20 +404,24 @@ class WebClientTestCase(unittest.TestCase):
             pageData,
             b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
             b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
-            b'<a href="/file">click here</a>\n    </body>\n</html>\n')
+            b'<a href="/file">click here</a>\n    </body>\n</html>\n',
+        )
 
     def test_encoding(self):
-        """ Test that non-standart body encoding matches
-        Content-Encoding header """
-        body = b'\xd0\x81\xd1\x8e\xd0\xaf'
-        dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r)
+        """Test that non-standart body encoding matches
+        Content-Encoding header"""
+        body = b"\xd0\x81\xd1\x8e\xd0\xaf"
+        dfd = getPage(
+            self.getURL("encoding"), body=body, response_transform=lambda r: r
+        )
         return dfd.addCallback(self._check_Encoding, body)
 
     def _check_Encoding(self, response, original_body):
-        content_encoding = to_unicode(response.headers[b'Content-Encoding'])
+        content_encoding = to_unicode(response.headers[b"Content-Encoding"])
         self.assertEqual(content_encoding, EncodingResource.out_encoding)
         self.assertEqual(
-            response.body.decode(content_encoding), to_unicode(original_body))
+            response.body.decode(content_encoding), to_unicode(original_body)
+        )
 
 
 class WebClientSSLTestCase(unittest.TestCase):
@@ -370,9 +429,11 @@ class WebClientSSLTestCase(unittest.TestCase):
 
     def _listen(self, site):
         return reactor.listenSSL(
-            0, site,
+            0,
+            site,
             contextFactory=self.context_factory or ssl_context_factory(),
-            interface="127.0.0.1")
+            interface="127.0.0.1",
+        )
 
     def getURL(self, path):
         return f"https://127.0.0.1:{self.portno}/{path}"
@@ -396,25 +457,34 @@ class WebClientSSLTestCase(unittest.TestCase):
     def testPayload(self):
         s = "0123456789" * 10
         return getPage(self.getURL("payload"), body=s).addCallback(
-            self.assertEqual, to_bytes(s))
+            self.assertEqual, to_bytes(s)
+        )
 
 
 class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
     # we try to use a cipher that is not enabled by default in OpenSSL
-    custom_ciphers = 'CAMELLIA256-SHA'
+    custom_ciphers = "CAMELLIA256-SHA"
     context_factory = ssl_context_factory(cipher_string=custom_ciphers)
 
     def testPayload(self):
         s = "0123456789" * 10
-        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers})
-        client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
+        settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers})
+        client_context_factory = create_instance(
+            ScrapyClientContextFactory, settings=settings, crawler=None
+        )
         return getPage(
             self.getURL("payload"), body=s, contextFactory=client_context_factory
         ).addCallback(self.assertEqual, to_bytes(s))
 
     def testPayloadDisabledCipher(self):
         s = "0123456789" * 10
-        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
-        client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
-        d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory)
+        settings = Settings(
+            {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"}
+        )
+        client_context_factory = create_instance(
+            ScrapyClientContextFactory, settings=settings, crawler=None
+        )
+        d = getPage(
+            self.getURL("payload"), body=s, contextFactory=client_context_factory
+        )
         return self.assertFailure(d, OpenSSL.SSL.Error)

From 44160552ef76188d895030d89f2659bcd114a80c Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Tue, 29 Nov 2022 11:40:14 -0300
Subject: [PATCH 02/15] adding .git-blame-ignore-revs file to skip code changes
 made by black formatter on blame

---
 .git-blame-ignore-revs | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .git-blame-ignore-revs

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
new file mode 100644
index 000000000..9d8f3d62e
--- /dev/null
+++ b/.git-blame-ignore-revs
@@ -0,0 +1,3 @@
+# .git-blame-ignore-revs
+# adding black formatter to all the code
+e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
\ No newline at end of file

From 1a3db8149224d6875a90513b52437fb1fa916e10 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Tue, 29 Nov 2022 15:06:54 -0300
Subject: [PATCH 03/15] ignoring E203 flake8 error ignoring implicit-str-concat
 pylint error fixing pylint comment on test_request_cb_kwargs.py

---
 .flake8                         | 2 +-
 pylintrc                        | 1 +
 tests/test_request_cb_kwargs.py | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.flake8 b/.flake8
index 0c64d009e..544d72956 100644
--- a/.flake8
+++ b/.flake8
@@ -1,7 +1,7 @@
 [flake8]
 
 max-line-length = 119
-ignore = W503
+ignore = W503, E203
 
 exclude =
     docs/conf.py
diff --git a/pylintrc b/pylintrc
index 0a2276fb8..94465fca4 100644
--- a/pylintrc
+++ b/pylintrc
@@ -28,6 +28,7 @@ disable=abstract-method,
         fixme,
         function-redefined,
         global-statement,
+        implicit-str-concat,
         import-error,
         import-outside-toplevel,
         import-self,
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 5a3dae103..454b68942 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -106,8 +106,8 @@ class KeywordArgumentsSpider(MockServerSpider):
             self.crawler.stats.inc_value("boolean_checks", 3)
         elif response.url.endswith("/general_without"):
             self.checks.append(
-                kwargs == {}
-            )  # pylint: disable=use-implicit-booleaness-not-comparison
+                kwargs == {}  # pylint: disable=use-implicit-booleaness-not-comparison
+            )
             self.crawler.stats.inc_value("boolean_checks")
 
     def parse_no_kwargs(self, response):

From 09dc4cf308f80c19d33e6f70d4e449eb85e2af29 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Thu, 22 Dec 2022 16:08:37 -0300
Subject: [PATCH 04/15] adding black to tox and defining line length on
 pyproject.toml

---
 pyproject.toml | 2 ++
 tox.ini        | 9 ++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 pyproject.toml

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..f5da62525
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,2 @@
+[tool.black]
+line-length = 119
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 4d0f0291b..913b8045e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = security,flake8,py
+envlist = security,flake8,py,black
 minversion = 1.7.0
 
 [testenv]
@@ -197,3 +197,10 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+[testenv:black]
+deps =
+    black==22.10.0
+commands =
+    black {posargs:--check .}
+    
\ No newline at end of file

From f5f2fc0ccd645514c0157e9116d3d645a23b3c60 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Mon, 26 Dec 2022 12:12:21 -0300
Subject: [PATCH 05/15] returning to black default line length

---
 pyproject.toml | 2 --
 1 file changed, 2 deletions(-)
 delete mode 100644 pyproject.toml

diff --git a/pyproject.toml b/pyproject.toml
deleted file mode 100644
index f5da62525..000000000
--- a/pyproject.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-[tool.black]
-line-length = 119
\ No newline at end of file

From c48accf7cae2c21ca204fe874725ebcb0c87af67 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Mon, 26 Dec 2022 13:51:27 -0300
Subject: [PATCH 06/15] adding pre-commit with config

---
 .pre-commit-config.yaml | 20 ++++++++++++++++++++
 tox.ini                 |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 .pre-commit-config.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..bbae805af
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,20 @@
+repos:
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.7.4
+  hooks:
+  - id: bandit
+    args: [-r, -c, .bandit.yml]
+- repo: https://github.com/PyCQA/flake8
+  rev: 6.0.0
+  hooks:
+  - id: flake8
+- repo: https://github.com/PyCQA/pylint
+  rev: v2.15.6
+  hooks:
+  - id: pylint
+    args: [conftest.py, docs, extras, scrapy, setup.py, tests]
+- repo: https://github.com/psf/black.git
+  rev: 22.12.0
+  hooks:
+  - id: black
+    args: [--line-length=79]
diff --git a/tox.ini b/tox.ini
index 913b8045e..e37ae4579 100644
--- a/tox.ini
+++ b/tox.ini
@@ -200,7 +200,7 @@ commands =
 
 [testenv:black]
 deps =
-    black==22.10.0
+    black==22.12.0
 commands =
     black {posargs:--check .}
     
\ No newline at end of file

From 4ee09861cf873cf1c003526f10853a9fc107e778 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Thu, 29 Dec 2022 08:20:48 -0300
Subject: [PATCH 07/15] adding black check for tox on github workflow

---
 .github/workflows/checks.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 8c1ae4bd3..90b6ade4a 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -28,6 +28,9 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: twinecheck
+        - python-version: "3.11"
+          env:
+            TOXENV: black
 
     steps:
     - uses: actions/checkout@v3

From 303f0a70fcf8067adf0a909c2096a5009162383a Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Thu, 29 Dec 2022 11:25:05 -0300
Subject: [PATCH 08/15] re applying black to the code with default line length

---
 scrapy/pipelines/images.py  |  36 ++++---
 scrapy/shell.py             |  11 ++-
 setup.py                    |  16 ++--
 tests/test_command_shell.py |  28 ++++--
 tests/test_http_response.py | 186 ++++++++++++++++++++++++++----------
 5 files changed, 196 insertions(+), 81 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index cfe25f903..adf3f5222 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -61,13 +61,17 @@ class ImagesPipeline(FilesPipeline):
                 "ImagesPipeline requires installing Pillow 4.0.0 or later"
             )
 
-        super().__init__(store_uri, settings=settings, download_func=download_func)
+        super().__init__(
+            store_uri, settings=settings, download_func=download_func
+        )
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
         resolve = functools.partial(
-            self._key_for_pipe, base_class_name="ImagesPipeline", settings=settings
+            self._key_for_pipe,
+            base_class_name="ImagesPipeline",
+            settings=settings,
         )
         self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
 
@@ -82,8 +86,12 @@ class ImagesPipeline(FilesPipeline):
         self.images_result_field = settings.get(
             resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD
         )
-        self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)
-        self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)
+        self.min_width = settings.getint(
+            resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH
+        )
+        self.min_height = settings.getint(
+            resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT
+        )
         self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)
 
         self._deprecated_convert_image = None
@@ -117,7 +125,9 @@ class ImagesPipeline(FilesPipeline):
 
     def image_downloaded(self, response, request, info, *, item=None):
         checksum = None
-        for path, image, buf in self.get_images(response, request, info, item=item):
+        for path, image, buf in self.get_images(
+            response, request, info, item=item
+        ):
             if checksum is None:
                 buf.seek(0)
                 checksum = md5sum(buf)
@@ -144,8 +154,8 @@ class ImagesPipeline(FilesPipeline):
             )
 
         if self._deprecated_convert_image is None:
-            self._deprecated_convert_image = "response_body" not in get_func_args(
-                self.convert_image
+            self._deprecated_convert_image = (
+                "response_body" not in get_func_args(self.convert_image)
             )
             if self._deprecated_convert_image:
                 warnings.warn(
@@ -181,8 +191,8 @@ class ImagesPipeline(FilesPipeline):
                 stacklevel=2,
             )
 
-        if image.format in ('PNG', 'WEBP') and image.mode == 'RGBA':
-            background = self._Image.new('RGBA', image.size, (255, 255, 255))
+        if image.format in ("PNG", "WEBP") and image.mode == "RGBA":
+            background = self._Image.new("RGBA", image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert("RGB")
         elif image.mode == "P":
@@ -216,13 +226,17 @@ class ImagesPipeline(FilesPipeline):
 
     def item_completed(self, results, item, info):
         with suppress(KeyError):
-            ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
+            ItemAdapter(item)[self.images_result_field] = [
+                x for ok, x in results if ok
+            ]
         return item
 
     def file_path(self, request, response=None, info=None, *, item=None):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f"full/{image_guid}.jpg"
 
-    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
+    def thumb_path(
+        self, request, thumb_id, response=None, info=None, *, item=None
+    ):
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 8e79908ab..c67334ca0 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -21,7 +21,10 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
-from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
+from scrapy.utils.reactor import (
+    is_asyncio_reactor_installed,
+    set_asyncio_event_loop,
+)
 
 
 class Shell:
@@ -37,7 +40,9 @@ class Shell:
         self.code = code
         self.vars = {}
 
-    def start(self, url=None, request=None, response=None, spider=None, redirect=True):
+    def start(
+        self, url=None, request=None, response=None, spider=None, redirect=True
+    ):
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
@@ -80,7 +85,7 @@ class Shell:
     def _schedule(self, request, spider):
         if is_asyncio_reactor_installed():
             # set the asyncio event loop for the current thread
-            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
+            event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
             set_asyncio_event_loop(event_loop_path)
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
diff --git a/setup.py b/setup.py
index 9ce93d964..5d1245f37 100644
--- a/setup.py
+++ b/setup.py
@@ -59,14 +59,14 @@ setup(
         "Source": "https://github.com/scrapy/scrapy",
         "Tracker": "https://github.com/scrapy/scrapy/issues",
     },
-    description='A high-level Web Crawling and Web Scraping framework',
-    long_description=open('README.rst', encoding="utf-8").read(),
-    author='Scrapy developers',
-    author_email='pablo@pablohoffman.com',
-    maintainer='Pablo Hoffman',
-    maintainer_email='pablo@pablohoffman.com',
-    license='BSD',
-    packages=find_packages(exclude=('tests', 'tests.*')),
+    description="A high-level Web Crawling and Web Scraping framework",
+    long_description=open("README.rst", encoding="utf-8").read(),
+    author="Scrapy developers",
+    author_email="pablo@pablohoffman.com",
+    maintainer="Pablo Hoffman",
+    maintainer_email="pablo@pablohoffman.com",
+    license="BSD",
+    packages=find_packages(exclude=("tests", "tests.*")),
     include_package_data=True,
     zip_safe=False,
     entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index aa5128bee..eced0e436 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -20,17 +20,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_response_body(self):
-        _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"])
+        _, out, _ = yield self.execute(
+            [self.url("/text"), "-c", "response.body"]
+        )
         assert b"Works" in out
 
     @defer.inlineCallbacks
     def test_response_type_text(self):
-        _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"])
+        _, out, _ = yield self.execute(
+            [self.url("/text"), "-c", "type(response)"]
+        )
         assert b"TextResponse" in out
 
     @defer.inlineCallbacks
     def test_response_type_html(self):
-        _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"])
+        _, out, _ = yield self.execute(
+            [self.url("/html"), "-c", "type(response)"]
+        )
         assert b"HtmlResponse" in out
 
     @defer.inlineCallbacks
@@ -48,7 +54,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_redirect(self):
-        _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"])
+        _, out, _ = yield self.execute(
+            [self.url("/redirect"), "-c", "response.url"]
+        )
         assert out.strip().endswith(b"/redirected")
 
     @defer.inlineCallbacks
@@ -92,7 +100,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_request_replace(self):
         url = self.url("/text")
-        code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))"
+        code = (
+            f"fetch('{url}') or fetch(response.request.replace(method='POST'))"
+        )
         errcode, out, _ = yield self.execute(["-c", code])
         self.assertEqual(errcode, 0, out)
 
@@ -123,14 +133,16 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         if NON_EXISTING_RESOLVABLE:
             raise unittest.SkipTest("Non-existing hosts are resolvable")
         url = "www.somedomainthatdoesntexi.st"
-        errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False)
+        errcode, out, err = yield self.execute(
+            [url, "-c", "item"], check_code=False
+        )
         self.assertEqual(errcode, 1, out or err)
-        self.assertIn(b'DNS lookup failed', err)
+        self.assertIn(b"DNS lookup failed", err)
 
     @defer.inlineCallbacks
     def test_shell_fetch_async(self):
         reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
-        url = self.url('/html')
+        url = self.url("/html")
         code = f"fetch('{url}')"
         args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"]
         _, _, err = yield self.execute(args, check_code=True)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 6e4971400..244d85c65 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -8,8 +8,14 @@ from w3lib import __version__ as w3lib_version
 from w3lib.encoding import resolve_encoding
 
 from scrapy.exceptions import NotSupported
-from scrapy.http import (Headers, HtmlResponse, Request, Response,
-                         TextResponse, XmlResponse)
+from scrapy.http import (
+    Headers,
+    HtmlResponse,
+    Request,
+    Response,
+    TextResponse,
+    XmlResponse,
+)
 from scrapy.link import Link
 from scrapy.selector import Selector
 from scrapy.utils.python import to_unicode
@@ -24,9 +30,13 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the constructor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(
-            isinstance(self.response_class("http://example.com/"), self.response_class)
+            isinstance(
+                self.response_class("http://example.com/"), self.response_class
+            )
+        )
+        self.assertRaises(
+            TypeError, self.response_class, b"http://example.com"
         )
-        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(
             isinstance(
@@ -60,7 +70,9 @@ class BaseResponseTest(unittest.TestCase):
 
         headers = {"foo": "bar"}
         body = b"a body"
-        r = self.response_class("http://www.example.com", headers=headers, body=body)
+        r = self.response_class(
+            "http://www.example.com", headers=headers, body=body
+        )
 
         assert r.headers is not headers
         self.assertEqual(r.headers[b"foo"], b"bar")
@@ -70,7 +82,10 @@ class BaseResponseTest(unittest.TestCase):
         r = self.response_class("http://www.example.com", status="301")
         self.assertEqual(r.status, 301)
         self.assertRaises(
-            ValueError, self.response_class, "http://example.com", status="lala200"
+            ValueError,
+            self.response_class,
+            "http://example.com",
+            status="lala200",
         )
 
     def test_copy(self):
@@ -84,7 +99,9 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(r1.body, r2.body)
 
         # make sure flags list is shallow copied
-        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
+        assert (
+            r1.flags is not r2.flags
+        ), "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
         # make sure headers attribute is shallow copied
@@ -111,7 +128,9 @@ class BaseResponseTest(unittest.TestCase):
 
     def test_unavailable_meta(self):
         r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"):
+        with self.assertRaisesRegex(
+            AttributeError, r"Response\.meta not available"
+        ):
             r1.meta
 
     def test_unavailable_cb_kwargs(self):
@@ -168,7 +187,9 @@ class BaseResponseTest(unittest.TestCase):
 
     def test_immutable_attributes(self):
         r = self.response_class("http://example.com")
-        self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
+        self.assertRaises(
+            AttributeError, setattr, r, "url", "http://example2.com"
+        )
         self.assertRaises(AttributeError, setattr, r, "body", "xxx")
 
     def test_urljoin(self):
@@ -192,7 +213,9 @@ class BaseResponseTest(unittest.TestCase):
     # Response.follow
 
     def test_follow_url_absolute(self):
-        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
+        self._assert_followed_url(
+            "http://foo.example.com", "http://foo.example.com"
+        )
 
     def test_follow_url_relative(self):
         self._assert_followed_url("foo", "http://example.com/foo")
@@ -212,8 +235,7 @@ class BaseResponseTest(unittest.TestCase):
         strict=True,
     )
     def test_follow_whitespace_url(self):
-        self._assert_followed_url('foo ',
-                                  'http://example.com/foo')
+        self._assert_followed_url("foo ", "http://example.com/foo")
 
     @mark.xfail(
         parse_version(w3lib_version) < parse_version("2.1.1"),
@@ -221,8 +243,9 @@ class BaseResponseTest(unittest.TestCase):
         strict=True,
     )
     def test_follow_whitespace_link(self):
-        self._assert_followed_url(Link('http://example.com/foo '),
-                                  'http://example.com/foo')
+        self._assert_followed_url(
+            Link("http://example.com/foo "), "http://example.com/foo"
+        )
 
     def test_follow_flags(self):
         res = self.response_class("http://example.com/")
@@ -320,7 +343,9 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(req.url, target_url)
         return req
 
-    def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
+    def _assert_followed_all_urls(
+        self, follow_obj, target_urls, response=None
+    ):
         if response is None:
             response = self._links_response()
         followed = response.follow_all(follow_obj)
@@ -360,16 +385,22 @@ class TextResponseTest(BaseResponseTest):
     def test_unicode_url(self):
         # instantiate with unicode url without encoding (should set default encoding)
         resp = self.response_class("http://www.example.com/")
-        self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
+        self._assert_response_encoding(
+            resp, self.response_class._DEFAULT_ENCODING
+        )
 
         # make sure urls are converted to str
-        resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
+        resp = self.response_class(
+            url="http://www.example.com/", encoding="utf-8"
+        )
         assert isinstance(resp.url, str)
 
         resp = self.response_class(
             url="http://www.example.com/price/\xa3", encoding="utf-8"
         )
-        self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
+        self.assertEqual(
+            resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")
+        )
         resp = self.response_class(
             url="http://www.example.com/price/\xa3", encoding="latin-1"
         )
@@ -378,7 +409,9 @@ class TextResponseTest(BaseResponseTest):
             "http://www.example.com/price/\xa3",
             headers={"Content-type": ["text/html; charset=utf-8"]},
         )
-        self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
+        self.assertEqual(
+            resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")
+        )
         resp = self.response_class(
             "http://www.example.com/price/\xa3",
             headers={"Content-type": ["text/html; charset=iso-8859-1"]},
@@ -466,7 +499,10 @@ class TextResponseTest(BaseResponseTest):
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
         self.assertRaises(
-            TypeError, self.response_class, "http://www.example.com", body="\xa3"
+            TypeError,
+            self.response_class,
+            "http://www.example.com",
+            body="\xa3",
         )
 
     def test_declared_encoding_invalid(self):
@@ -482,7 +518,9 @@ class TextResponseTest(BaseResponseTest):
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
         r = self.response_class(
-            "http://www.example.com", body=b"\xff\xfeh\x00i\x00", encoding="utf-16"
+            "http://www.example.com",
+            body=b"\xff\xfeh\x00i\x00",
+            encoding="utf-16",
         )
         self._assert_response_values(r, "utf-16", "hi")
 
@@ -531,7 +569,9 @@ class TextResponseTest(BaseResponseTest):
     def test_replace_wrong_encoding(self):
         """Test invalid chars are replaced properly"""
         r = self.response_class(
-            "http://www.example.com", encoding="utf-8", body=b"PREFIX\xe3\xabSUFFIX"
+            "http://www.example.com",
+            encoding="utf-8",
+            body=b"PREFIX\xe3\xabSUFFIX",
         )
         # XXX: Policy for replacing invalid chars may suffer minor variations
         # but it should always contain the unicode replacement char ('\ufffd')
@@ -541,7 +581,9 @@ class TextResponseTest(BaseResponseTest):
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class(
-            "http://example.com", encoding="utf-8", body=b"\xf0<span>value</span>"
+            "http://example.com",
+            encoding="utf-8",
+            body=b"\xf0<span>value</span>",
         )
         assert "<span>value</span>" in r.text, repr(r.text)
 
@@ -555,13 +597,17 @@ class TextResponseTest(BaseResponseTest):
 
         self.assertIsInstance(response.selector, Selector)
         self.assertEqual(response.selector.type, "html")
-        self.assertIs(response.selector, response.selector)  # property is cached
+        self.assertIs(
+            response.selector, response.selector
+        )  # property is cached
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
             response.selector.xpath("//title/text()").getall(), ["Some page"]
         )
-        self.assertEqual(response.selector.css("title::text").getall(), ["Some page"])
+        self.assertEqual(
+            response.selector.css("title::text").getall(), ["Some page"]
+        )
         self.assertEqual(response.selector.re("Some (.*)</title>"), ["page"])
 
     def test_selector_shortcuts(self):
@@ -601,23 +647,23 @@ class TextResponseTest(BaseResponseTest):
     def test_urljoin_with_base_url(self):
         """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
         body = b'<html><body><base href="https://example.net"></body></html>'
-        joined = self.response_class("http://www.example.com", body=body).urljoin(
-            "/test"
-        )
+        joined = self.response_class(
+            "http://www.example.com", body=body
+        ).urljoin("/test")
         absolute = "https://example.net/test"
         self.assertEqual(joined, absolute)
 
         body = b'<html><body><base href="/elsewhere"></body></html>'
-        joined = self.response_class("http://www.example.com", body=body).urljoin(
-            "test"
-        )
+        joined = self.response_class(
+            "http://www.example.com", body=body
+        ).urljoin("test")
         absolute = "http://www.example.com/test"
         self.assertEqual(joined, absolute)
 
         body = b'<html><body><base href="/elsewhere/"></body></html>'
-        joined = self.response_class("http://www.example.com", body=body).urljoin(
-            "test"
-        )
+        joined = self.response_class(
+            "http://www.example.com", body=body
+        ).urljoin("test")
         absolute = "http://www.example.com/elsewhere/test"
         self.assertEqual(joined, absolute)
 
@@ -654,12 +700,17 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_selector_list(self):
         resp = self._links_response()
-        self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a"))
+        self.assertRaisesRegex(
+            ValueError, "SelectorList", resp.follow, resp.css("a")
+        )
 
     def test_follow_selector_invalid(self):
         resp = self._links_response()
         self.assertRaisesRegex(
-            ValueError, "Unsupported", resp.follow, resp.xpath("count(//div)")[0]
+            ValueError,
+            "Unsupported",
+            resp.follow,
+            resp.xpath("count(//div)")[0],
         )
 
     def test_follow_selector_attribute(self):
@@ -672,7 +723,9 @@ class TextResponseTest(BaseResponseTest):
             url="http://example.com",
             body=b"<html><body><a name=123>click me</a></body></html>",
         )
-        self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0])
+        self.assertRaisesRegex(
+            ValueError, "no href", resp.follow, resp.css("a")[0]
+        )
 
     def test_follow_whitespace_selector(self):
         resp = self.response_class(
@@ -683,7 +736,9 @@ class TextResponseTest(BaseResponseTest):
             resp.css("a")[0], "http://example.com/foo", response=resp
         )
         self._assert_followed_url(
-            resp.css("a::attr(href)")[0], "http://example.com/foo", response=resp
+            resp.css("a::attr(href)")[0],
+            "http://example.com/foo",
+            response=resp,
         )
 
     def test_follow_encoding(self):
@@ -737,7 +792,9 @@ class TextResponseTest(BaseResponseTest):
             "http://example.com/innertag.html",
         ]
         response = self._links_response()
-        extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
+        extracted = [
+            r.url for r in response.follow_all(css='a[href*="example.com"]')
+        ]
         self.assertEqual(expected, extracted)
 
     def test_follow_all_css_skip_invalid(self):
@@ -749,7 +806,9 @@ class TextResponseTest(BaseResponseTest):
         response = self._links_response_no_href()
         extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
         self.assertEqual(expected, extracted1)
-        extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
+        extracted2 = [
+            r.url for r in response.follow_all(response.css(".pagination a"))
+        ]
         self.assertEqual(expected, extracted2)
 
     def test_follow_all_xpath(self):
@@ -758,7 +817,9 @@ class TextResponseTest(BaseResponseTest):
             "http://example.com/innertag.html",
         ]
         response = self._links_response()
-        extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
+        extracted = response.follow_all(
+            xpath='//a[contains(@href, "example.com")]'
+        )
         self.assertEqual(expected, [r.url for r in extracted])
 
     def test_follow_all_xpath_skip_invalid(self):
@@ -769,12 +830,15 @@ class TextResponseTest(BaseResponseTest):
         ]
         response = self._links_response_no_href()
         extracted1 = [
-            r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
+            r.url
+            for r in response.follow_all(xpath='//div[@id="pagination"]/a')
         ]
         self.assertEqual(expected, extracted1)
         extracted2 = [
             r.url
-            for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
+            for r in response.follow_all(
+                response.xpath('//div[@id="pagination"]/a')
+            )
         ]
         self.assertEqual(expected, extracted2)
 
@@ -788,11 +852,15 @@ class TextResponseTest(BaseResponseTest):
 
     def test_json_response(self):
         json_body = b"""{"ip": "109.187.217.200"}"""
-        json_response = self.response_class("http://www.example.com", body=json_body)
+        json_response = self.response_class(
+            "http://www.example.com", body=json_body
+        )
         self.assertEqual(json_response.json(), {"ip": "109.187.217.200"})
 
         text_body = b"""<html><body>text</body></html>"""
-        text_response = self.response_class("http://www.example.com", body=text_body)
+        text_response = self.response_class(
+            "http://www.example.com", body=text_body
+        )
         with self.assertRaises(ValueError):
             text_response.json()
 
@@ -859,7 +927,9 @@ class XmlResponseTest(TextResponseTest):
     def test_xml_encoding(self):
         body = b"<xml></xml>"
         r1 = self.response_class("http://www.example.com", body=body)
-        self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)
+        self._assert_response_values(
+            r1, self.response_class._DEFAULT_ENCODING, body
+        )
 
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
         r2 = self.response_class("http://www.example.com", body=body)
@@ -867,7 +937,9 @@ class XmlResponseTest(TextResponseTest):
 
         # make sure replace() preserves the explicit encoding passed in the __init__ method
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
-        r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8")
+        r3 = self.response_class(
+            "http://www.example.com", body=body, encoding="utf-8"
+        )
         body2 = b"New body"
         r4 = r3.replace(body=body2)
         self._assert_response_values(r4, "utf-8", body2)
@@ -889,10 +961,14 @@ class XmlResponseTest(TextResponseTest):
 
         self.assertIsInstance(response.selector, Selector)
         self.assertEqual(response.selector.type, "xml")
-        self.assertIs(response.selector, response.selector)  # property is cached
+        self.assertIs(
+            response.selector, response.selector
+        )  # property is cached
         self.assertIs(response.selector.response, response)
 
-        self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"])
+        self.assertEqual(
+            response.selector.xpath("//elem/text()").getall(), ["value"]
+        )
 
     def test_selector_shortcuts(self):
         body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
@@ -944,7 +1020,11 @@ class CustomResponseTest(TextResponseTest):
     def test_copy(self):
         super().test_copy()
         r1 = self.response_class(
-            url="https://example.org", status=200, foo="foo", bar="bar", lost="lost"
+            url="https://example.org",
+            status=200,
+            foo="foo",
+            bar="bar",
+            lost="lost",
         )
         r2 = r1.copy()
         self.assertIsInstance(r2, self.response_class)
@@ -956,7 +1036,11 @@ class CustomResponseTest(TextResponseTest):
     def test_replace(self):
         super().test_replace()
         r1 = self.response_class(
-            url="https://example.org", status=200, foo="foo", bar="bar", lost="lost"
+            url="https://example.org",
+            status=200,
+            foo="foo",
+            bar="bar",
+            lost="lost",
         )
 
         r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")

From ef6eb48b2d6efa0a92dc52f80dd4f424058d6772 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Thu, 29 Dec 2022 11:27:41 -0300
Subject: [PATCH 09/15] ignoring changes made by black

---
 .git-blame-ignore-revs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index 9d8f3d62e..3e63aa8bd 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -1,3 +1,5 @@
 # .git-blame-ignore-revs
 # adding black formatter to all the code
-e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
\ No newline at end of file
+e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
+# re applying black to the code with default line length
+303f0a70fcf8067adf0a909c2096a5009162383a
\ No newline at end of file

From c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Thu, 29 Dec 2022 12:46:57 -0300
Subject: [PATCH 10/15] reaplying black again and removing line length on
 pre-commit black config

---
 .pre-commit-config.yaml     |   1 -
 scrapy/pipelines/images.py  |  28 +++-----
 scrapy/shell.py             |   4 +-
 tests/test_command_shell.py |  28 ++------
 tests/test_http_response.py | 125 +++++++++++-------------------------
 5 files changed, 52 insertions(+), 134 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index bbae805af..b93a73453 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -17,4 +17,3 @@ repos:
   rev: 22.12.0
   hooks:
   - id: black
-    args: [--line-length=79]
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index adf3f5222..fcecb5fbb 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -61,9 +61,7 @@ class ImagesPipeline(FilesPipeline):
                 "ImagesPipeline requires installing Pillow 4.0.0 or later"
             )
 
-        super().__init__(
-            store_uri, settings=settings, download_func=download_func
-        )
+        super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
@@ -86,12 +84,8 @@ class ImagesPipeline(FilesPipeline):
         self.images_result_field = settings.get(
             resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD
         )
-        self.min_width = settings.getint(
-            resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH
-        )
-        self.min_height = settings.getint(
-            resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT
-        )
+        self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)
+        self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)
         self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)
 
         self._deprecated_convert_image = None
@@ -125,9 +119,7 @@ class ImagesPipeline(FilesPipeline):
 
     def image_downloaded(self, response, request, info, *, item=None):
         checksum = None
-        for path, image, buf in self.get_images(
-            response, request, info, item=item
-        ):
+        for path, image, buf in self.get_images(response, request, info, item=item):
             if checksum is None:
                 buf.seek(0)
                 checksum = md5sum(buf)
@@ -154,8 +146,8 @@ class ImagesPipeline(FilesPipeline):
             )
 
         if self._deprecated_convert_image is None:
-            self._deprecated_convert_image = (
-                "response_body" not in get_func_args(self.convert_image)
+            self._deprecated_convert_image = "response_body" not in get_func_args(
+                self.convert_image
             )
             if self._deprecated_convert_image:
                 warnings.warn(
@@ -226,17 +218,13 @@ class ImagesPipeline(FilesPipeline):
 
     def item_completed(self, results, item, info):
         with suppress(KeyError):
-            ItemAdapter(item)[self.images_result_field] = [
-                x for ok, x in results if ok
-            ]
+            ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None, *, item=None):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f"full/{image_guid}.jpg"
 
-    def thumb_path(
-        self, request, thumb_id, response=None, info=None, *, item=None
-    ):
+    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
diff --git a/scrapy/shell.py b/scrapy/shell.py
index c67334ca0..a42dedc93 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -40,9 +40,7 @@ class Shell:
         self.code = code
         self.vars = {}
 
-    def start(
-        self, url=None, request=None, response=None, spider=None, redirect=True
-    ):
+    def start(self, url=None, request=None, response=None, spider=None, redirect=True):
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index eced0e436..7e99a7296 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -20,23 +20,17 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_response_body(self):
-        _, out, _ = yield self.execute(
-            [self.url("/text"), "-c", "response.body"]
-        )
+        _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"])
         assert b"Works" in out
 
     @defer.inlineCallbacks
     def test_response_type_text(self):
-        _, out, _ = yield self.execute(
-            [self.url("/text"), "-c", "type(response)"]
-        )
+        _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"])
         assert b"TextResponse" in out
 
     @defer.inlineCallbacks
     def test_response_type_html(self):
-        _, out, _ = yield self.execute(
-            [self.url("/html"), "-c", "type(response)"]
-        )
+        _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"])
         assert b"HtmlResponse" in out
 
     @defer.inlineCallbacks
@@ -54,9 +48,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_redirect(self):
-        _, out, _ = yield self.execute(
-            [self.url("/redirect"), "-c", "response.url"]
-        )
+        _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"])
         assert out.strip().endswith(b"/redirected")
 
     @defer.inlineCallbacks
@@ -100,9 +92,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_request_replace(self):
         url = self.url("/text")
-        code = (
-            f"fetch('{url}') or fetch(response.request.replace(method='POST'))"
-        )
+        code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))"
         errcode, out, _ = yield self.execute(["-c", code])
         self.assertEqual(errcode, 0, out)
 
@@ -133,9 +123,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         if NON_EXISTING_RESOLVABLE:
             raise unittest.SkipTest("Non-existing hosts are resolvable")
         url = "www.somedomainthatdoesntexi.st"
-        errcode, out, err = yield self.execute(
-            [url, "-c", "item"], check_code=False
-        )
+        errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b"DNS lookup failed", err)
 
@@ -146,6 +134,4 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         code = f"fetch('{url}')"
         args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"]
         _, _, err = yield self.execute(args, check_code=True)
-        self.assertNotIn(
-            b"RuntimeError: There is no current event loop in thread", err
-        )
+        self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 244d85c65..ce32092c1 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -30,13 +30,9 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the constructor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(
-            isinstance(
-                self.response_class("http://example.com/"), self.response_class
-            )
-        )
-        self.assertRaises(
-            TypeError, self.response_class, b"http://example.com"
+            isinstance(self.response_class("http://example.com/"), self.response_class)
         )
+        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(
             isinstance(
@@ -70,9 +66,7 @@ class BaseResponseTest(unittest.TestCase):
 
         headers = {"foo": "bar"}
         body = b"a body"
-        r = self.response_class(
-            "http://www.example.com", headers=headers, body=body
-        )
+        r = self.response_class("http://www.example.com", headers=headers, body=body)
 
         assert r.headers is not headers
         self.assertEqual(r.headers[b"foo"], b"bar")
@@ -99,9 +93,7 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(r1.body, r2.body)
 
         # make sure flags list is shallow copied
-        assert (
-            r1.flags is not r2.flags
-        ), "flags must be a shallow copy, not identical"
+        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
         # make sure headers attribute is shallow copied
@@ -128,9 +120,7 @@ class BaseResponseTest(unittest.TestCase):
 
     def test_unavailable_meta(self):
         r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with self.assertRaisesRegex(
-            AttributeError, r"Response\.meta not available"
-        ):
+        with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"):
             r1.meta
 
     def test_unavailable_cb_kwargs(self):
@@ -187,9 +177,7 @@ class BaseResponseTest(unittest.TestCase):
 
     def test_immutable_attributes(self):
         r = self.response_class("http://example.com")
-        self.assertRaises(
-            AttributeError, setattr, r, "url", "http://example2.com"
-        )
+        self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
         self.assertRaises(AttributeError, setattr, r, "body", "xxx")
 
     def test_urljoin(self):
@@ -213,9 +201,7 @@ class BaseResponseTest(unittest.TestCase):
     # Response.follow
 
     def test_follow_url_absolute(self):
-        self._assert_followed_url(
-            "http://foo.example.com", "http://foo.example.com"
-        )
+        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
 
     def test_follow_url_relative(self):
         self._assert_followed_url("foo", "http://example.com/foo")
@@ -343,9 +329,7 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(req.url, target_url)
         return req
 
-    def _assert_followed_all_urls(
-        self, follow_obj, target_urls, response=None
-    ):
+    def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
         if response is None:
             response = self._links_response()
         followed = response.follow_all(follow_obj)
@@ -385,22 +369,16 @@ class TextResponseTest(BaseResponseTest):
     def test_unicode_url(self):
         # instantiate with unicode url without encoding (should set default encoding)
         resp = self.response_class("http://www.example.com/")
-        self._assert_response_encoding(
-            resp, self.response_class._DEFAULT_ENCODING
-        )
+        self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
 
         # make sure urls are converted to str
-        resp = self.response_class(
-            url="http://www.example.com/", encoding="utf-8"
-        )
+        resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
         assert isinstance(resp.url, str)
 
         resp = self.response_class(
             url="http://www.example.com/price/\xa3", encoding="utf-8"
         )
-        self.assertEqual(
-            resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")
-        )
+        self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
         resp = self.response_class(
             url="http://www.example.com/price/\xa3", encoding="latin-1"
         )
@@ -409,9 +387,7 @@ class TextResponseTest(BaseResponseTest):
             "http://www.example.com/price/\xa3",
             headers={"Content-type": ["text/html; charset=utf-8"]},
         )
-        self.assertEqual(
-            resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")
-        )
+        self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3"))
         resp = self.response_class(
             "http://www.example.com/price/\xa3",
             headers={"Content-type": ["text/html; charset=iso-8859-1"]},
@@ -597,17 +573,13 @@ class TextResponseTest(BaseResponseTest):
 
         self.assertIsInstance(response.selector, Selector)
         self.assertEqual(response.selector.type, "html")
-        self.assertIs(
-            response.selector, response.selector
-        )  # property is cached
+        self.assertIs(response.selector, response.selector)  # property is cached
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
             response.selector.xpath("//title/text()").getall(), ["Some page"]
         )
-        self.assertEqual(
-            response.selector.css("title::text").getall(), ["Some page"]
-        )
+        self.assertEqual(response.selector.css("title::text").getall(), ["Some page"])
         self.assertEqual(response.selector.re("Some (.*)</title>"), ["page"])
 
     def test_selector_shortcuts(self):
@@ -647,23 +619,23 @@ class TextResponseTest(BaseResponseTest):
     def test_urljoin_with_base_url(self):
         """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
         body = b'<html><body><base href="https://example.net"></body></html>'
-        joined = self.response_class(
-            "http://www.example.com", body=body
-        ).urljoin("/test")
+        joined = self.response_class("http://www.example.com", body=body).urljoin(
+            "/test"
+        )
         absolute = "https://example.net/test"
         self.assertEqual(joined, absolute)
 
         body = b'<html><body><base href="/elsewhere"></body></html>'
-        joined = self.response_class(
-            "http://www.example.com", body=body
-        ).urljoin("test")
+        joined = self.response_class("http://www.example.com", body=body).urljoin(
+            "test"
+        )
         absolute = "http://www.example.com/test"
         self.assertEqual(joined, absolute)
 
         body = b'<html><body><base href="/elsewhere/"></body></html>'
-        joined = self.response_class(
-            "http://www.example.com", body=body
-        ).urljoin("test")
+        joined = self.response_class("http://www.example.com", body=body).urljoin(
+            "test"
+        )
         absolute = "http://www.example.com/elsewhere/test"
         self.assertEqual(joined, absolute)
 
@@ -700,9 +672,7 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_selector_list(self):
         resp = self._links_response()
-        self.assertRaisesRegex(
-            ValueError, "SelectorList", resp.follow, resp.css("a")
-        )
+        self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a"))
 
     def test_follow_selector_invalid(self):
         resp = self._links_response()
@@ -723,9 +693,7 @@ class TextResponseTest(BaseResponseTest):
             url="http://example.com",
             body=b"<html><body><a name=123>click me</a></body></html>",
         )
-        self.assertRaisesRegex(
-            ValueError, "no href", resp.follow, resp.css("a")[0]
-        )
+        self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0])
 
     def test_follow_whitespace_selector(self):
         resp = self.response_class(
@@ -792,9 +760,7 @@ class TextResponseTest(BaseResponseTest):
             "http://example.com/innertag.html",
         ]
         response = self._links_response()
-        extracted = [
-            r.url for r in response.follow_all(css='a[href*="example.com"]')
-        ]
+        extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
         self.assertEqual(expected, extracted)
 
     def test_follow_all_css_skip_invalid(self):
@@ -806,9 +772,7 @@ class TextResponseTest(BaseResponseTest):
         response = self._links_response_no_href()
         extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
         self.assertEqual(expected, extracted1)
-        extracted2 = [
-            r.url for r in response.follow_all(response.css(".pagination a"))
-        ]
+        extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
         self.assertEqual(expected, extracted2)
 
     def test_follow_all_xpath(self):
@@ -817,9 +781,7 @@ class TextResponseTest(BaseResponseTest):
             "http://example.com/innertag.html",
         ]
         response = self._links_response()
-        extracted = response.follow_all(
-            xpath='//a[contains(@href, "example.com")]'
-        )
+        extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
         self.assertEqual(expected, [r.url for r in extracted])
 
     def test_follow_all_xpath_skip_invalid(self):
@@ -830,15 +792,12 @@ class TextResponseTest(BaseResponseTest):
         ]
         response = self._links_response_no_href()
         extracted1 = [
-            r.url
-            for r in response.follow_all(xpath='//div[@id="pagination"]/a')
+            r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
         ]
         self.assertEqual(expected, extracted1)
         extracted2 = [
             r.url
-            for r in response.follow_all(
-                response.xpath('//div[@id="pagination"]/a')
-            )
+            for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
         ]
         self.assertEqual(expected, extracted2)
 
@@ -852,15 +811,11 @@ class TextResponseTest(BaseResponseTest):
 
     def test_json_response(self):
         json_body = b"""{"ip": "109.187.217.200"}"""
-        json_response = self.response_class(
-            "http://www.example.com", body=json_body
-        )
+        json_response = self.response_class("http://www.example.com", body=json_body)
         self.assertEqual(json_response.json(), {"ip": "109.187.217.200"})
 
         text_body = b"""<html><body>text</body></html>"""
-        text_response = self.response_class(
-            "http://www.example.com", body=text_body
-        )
+        text_response = self.response_class("http://www.example.com", body=text_body)
         with self.assertRaises(ValueError):
             text_response.json()
 
@@ -927,9 +882,7 @@ class XmlResponseTest(TextResponseTest):
     def test_xml_encoding(self):
         body = b"<xml></xml>"
         r1 = self.response_class("http://www.example.com", body=body)
-        self._assert_response_values(
-            r1, self.response_class._DEFAULT_ENCODING, body
-        )
+        self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)
 
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
         r2 = self.response_class("http://www.example.com", body=body)
@@ -937,9 +890,7 @@ class XmlResponseTest(TextResponseTest):
 
         # make sure replace() preserves the explicit encoding passed in the __init__ method
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
-        r3 = self.response_class(
-            "http://www.example.com", body=body, encoding="utf-8"
-        )
+        r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8")
         body2 = b"New body"
         r4 = r3.replace(body=body2)
         self._assert_response_values(r4, "utf-8", body2)
@@ -961,14 +912,10 @@ class XmlResponseTest(TextResponseTest):
 
         self.assertIsInstance(response.selector, Selector)
         self.assertEqual(response.selector.type, "xml")
-        self.assertIs(
-            response.selector, response.selector
-        )  # property is cached
+        self.assertIs(response.selector, response.selector)  # property is cached
         self.assertIs(response.selector.response, response)
 
-        self.assertEqual(
-            response.selector.xpath("//elem/text()").getall(), ["value"]
-        )
+        self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"])
 
     def test_selector_shortcuts(self):
         body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'

From 1ebcd86c0466589c9f1afa9b7d5da051a90c5cbc Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Thu, 29 Dec 2022 12:51:05 -0300
Subject: [PATCH 11/15] ignoring last changes made by black for blame

---
 .git-blame-ignore-revs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index 3e63aa8bd..dbcebfa0a 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -2,4 +2,6 @@
 # adding black formatter to all the code
 e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
 # re applying black to the code with default line length
-303f0a70fcf8067adf0a909c2096a5009162383a
\ No newline at end of file
+303f0a70fcf8067adf0a909c2096a5009162383a
+# reaplying black again and removing line length on pre-commit black config
+c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
\ No newline at end of file

From 482a0b79e3e5ca274cf30584eb8a29c1b79791a8 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Tue, 17 Jan 2023 14:26:12 -0300
Subject: [PATCH 12/15] explaining pre-commit on documentation

---
 docs/contributing.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 9cfe10012..dca635f15 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -179,6 +179,25 @@ Scrapy:
   See https://help.github.com/en/github/using-git/setting-your-username-in-git for
   setup instructions.
 
+.. _pre-commit:
+
+Pre-commit
+==========
+
+Pre-commit is a tool that allows developers to specify a set of checks to be run 
+automatically every time they make a commit. This can include code style checks, 
+linting, and automated tests. 
+The checks are defined in a configuration file called .pre-commit-config.yaml.
+
+By using pre-commit, developers can ensure that their code adheres to a consistent 
+style and passes certain tests before they make a commit. 
+This can help catch errors early in the development process and prevent them 
+from being pushed to the main branch.
+
+To use pre-commit, developers first need to install it on their local machine. 
+Once it is installed, they can specify the checks they want to run in the .pre-commit-config.yaml file. 
+After that, pre-commit will run automatically every time they make a commit.
+
 .. _documentation-policies:
 
 Documentation policies

From 23e8b553b4db0c33645f371fd161470545733320 Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Wed, 18 Jan 2023 11:42:30 -0300
Subject: [PATCH 13/15] fixing docs with suggestions

---
 docs/contributing.rst | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index dca635f15..f7f1218c8 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -169,10 +169,10 @@ Coding style
 Please follow these coding conventions when writing code for inclusion in
 Scrapy:
 
-* Unless otherwise specified, follow :pep:`8`.
-
-* It's OK to use lines longer than 79 chars if it improves the code
-  readability.
+* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting. 
+  There is a hook in the pre-commit config
+  that will automatically format your code before every commit. You can also
+  run black manually with ``tox -e black``.
 
 * Don't put your name in the code you contribute; git provides enough
   metadata to identify author of the code.
@@ -184,19 +184,21 @@ Scrapy:
 Pre-commit
 ==========
 
-Pre-commit is a tool that allows developers to specify a set of checks to be run 
-automatically every time they make a commit. This can include code style checks, 
-linting, and automated tests. 
-The checks are defined in a configuration file called .pre-commit-config.yaml.
+We use `pre-commit`_ to automatically address simple code issues before every 
+commit.
 
-By using pre-commit, developers can ensure that their code adheres to a consistent 
-style and passes certain tests before they make a commit. 
-This can help catch errors early in the development process and prevent them 
-from being pushed to the main branch.
+Before you start writing a patch:
+#.  `Install pre-commit <https://pre-commit.com/#installation>`_.
+#.  On the root of your local clone of the Scrapy repository, run the following command:
+    .. code-block:: bash
+      
+      pre-commit install
 
-To use pre-commit, developers first need to install it on their local machine. 
-Once it is installed, they can specify the checks they want to run in the .pre-commit-config.yaml file. 
-After that, pre-commit will run automatically every time they make a commit.
+Now our pre-commit hooks will run every time you create a Git commit. Upon 
+finding issues, pre-commit hooks aborts your commit, and they either fix 
+the corresponding issues automatically or only report them to you. If they fix 
+the issues automatically, creating your commit again should succeed. Otherwise, 
+you may need to address the corresponding issues manually first.
 
 .. _documentation-policies:
 

From 8ee48174717591bbf74497fded060d7ae83d67ba Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Fri, 20 Jan 2023 10:54:46 -0300
Subject: [PATCH 14/15] reaplying black, fixing conflicts and ignoring bandit
 checks on test directory

---
 .bandit.yml                                   |   1 +
 .github/workflows/checks.yml                  |   4 +-
 .github/workflows/publish.yml                 |   4 +-
 .github/workflows/tests-ubuntu.yml            |   5 +-
 docs/news.rst                                 |  18 +--
 docs/topics/settings.rst                      |  32 +++--
 docs/topics/spiders.rst                       |   2 +-
 extras/qpsclient.py                           |   5 +-
 scrapy/cmdline.py                             |   2 +-
 scrapy/core/downloader/contextfactory.py      |   4 +-
 scrapy/core/downloader/tls.py                 |   4 +-
 scrapy/core/downloader/webclient.py           |   2 +-
 scrapy/core/http2/stream.py                   |   2 +-
 scrapy/extensions/feedexport.py               |   2 +-
 scrapy/extensions/httpcache.py                |   2 +-
 scrapy/http/request/form.py                   |   2 +-
 scrapy/linkextractors/lxmlhtml.py             |   7 +-
 scrapy/pipelines/images.py                    |   4 +-
 scrapy/shell.py                               |   4 +
 scrapy/utils/console.py                       |  15 +--
 scrapy/utils/defer.py                         |  10 +-
 scrapy/utils/reactor.py                       |  28 +++-
 scrapy/utils/response.py                      |   2 +-
 scrapy/utils/ssl.py                           |  12 +-
 scrapy/utils/url.py                           |   2 +-
 sep/sep-016.rst                               |   2 +-
 sep/sep-018.rst                               |  12 +-
 sep/sep-021.rst                               |   2 +-
 setup.py                                      |   2 +-
 tests/mockserver.py                           |   3 +-
 tests/pipelines.py                            |   2 +-
 .../link_extractor/linkextractor.html         |   1 +
 tests/test_commands.py                        |   2 +-
 tests/test_crawl.py                           |   4 +-
 tests/test_downloader_handlers.py             |  21 ++-
 tests/test_downloadermiddleware_useragent.py  |   2 +-
 tests/test_engine.py                          |   2 +-
 tests/test_feedexport.py                      | 125 +++++++++++++++++-
 tests/test_http_request.py                    |   8 +-
 tests/test_http_response.py                   |   3 +-
 tests/test_linkextractors.py                  |   8 ++
 tests/test_pipeline_images.py                 |  10 +-
 tests/test_request_attribute_binding.py       |  16 +--
 tests/test_spidermiddleware_referer.py        |  16 +--
 tests/test_utils_asyncio.py                   |   5 +
 tests/test_utils_deprecate.py                 |   2 +-
 tests/test_utils_display.py                   |  24 +++-
 tests/test_utils_python.py                    |   2 +-
 tox.ini                                       |  16 +--
 49 files changed, 326 insertions(+), 139 deletions(-)

diff --git a/.bandit.yml b/.bandit.yml
index 41f1bb597..c8e84cc2e 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -17,3 +17,4 @@ skips:
 - B503
 - B603
 - B605
+exclude_dirs: ['tests']
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 90b6ade4a..740092dab 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -14,9 +14,7 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: flake8
-        # Pylint requires installing reppy, which does not support Python 3.9
-        # https://github.com/seomoz/reppy/issues/122
-        - python-version: 3.8
+        - python-version: "3.11"
           env:
             TOXENV: pylint
         - python-version: 3.7
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 991b0b6e8..eee9a4f02 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -24,8 +24,8 @@ jobs:
     - name: Publish to PyPI
       if: steps.check-release-tag.outputs.release_tag == 'true'
       run: |
-        pip install --upgrade setuptools wheel twine
-        python setup.py sdist bdist_wheel
+        pip install --upgrade build twine
+        python -m build
         export TWINE_USERNAME=__token__
         export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
         twine upload dist/*
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 9c3ce8115..8fcf90a18 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -38,10 +38,7 @@ jobs:
           env:
             TOXENV: pypy3-pinned
 
-        # extras
-        # extra-deps includes reppy, which does not support Python 3.9
-        # https://github.com/seomoz/reppy/issues/122
-        - python-version: 3.8
+        - python-version: "3.11"
           env:
             TOXENV: extra-deps
 
diff --git a/docs/news.rst b/docs/news.rst
index c97de0ed8..07264827b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -4700,7 +4700,7 @@ Scrapy 0.22.1 (released 2014-02-08)
 - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
 - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
 - Fix tests for Travis-CI build (:commit:`76c7e20`)
-- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
+- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
 - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
 - Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
 - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
@@ -4725,7 +4725,7 @@ Enhancements
 - [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
   To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
 - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
-- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
+- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`)
 - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
 - Selectors register EXSLT namespaces by default (:issue:`472`)
 - Unify item loaders similar to selectors renaming (:issue:`461`)
@@ -4905,7 +4905,7 @@ Scrapy 0.18.0 (released 2013-08-09)
 -----------------------------------
 
 - Lot of improvements to testsuite run using Tox, including a way to test on pypi
-- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
+- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`)
 - Use lxml recover option to parse sitemaps (:issue:`347`)
 - Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
 - Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
@@ -4939,8 +4939,8 @@ Scrapy 0.18.0 (released 2013-08-09)
 - Added ``--pdb`` option to ``scrapy`` command line tool
 - Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
 - Several improvements to spider contracts
-- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
-- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
+- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections,
+- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62
 - added from_crawler method to spiders
 - added system tests with mock server
 - more improvements to macOS compatibility (thanks Alex Cepoi)
@@ -5082,7 +5082,7 @@ Scrapy changes:
 - promoted :ref:`topics-djangoitem` to main contrib
 - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
 - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
-- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
+- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
 - removed signal: ``scrapy.mail.mail_sent``
 - removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
 - DBM is now the default storage backend for HTTP cache middleware
@@ -5148,7 +5148,7 @@ Scrapy 0.14
 New features and settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Support for `AJAX crawleable urls`_
+- Support for `AJAX crawlable urls`_
 - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
 - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
 - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
@@ -5408,7 +5408,7 @@ Backward-incompatible changes
 - Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
 - Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
 - Refactored HTTP Cache middleware
-- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
+- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
 - Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
 - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
 - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
@@ -5419,7 +5419,7 @@ Scrapy 0.7
 First release of Scrapy.
 
 
-.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
+.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
 .. _botocore: https://github.com/boto/botocore
 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 022265992..e147a943b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -636,19 +636,30 @@ DOWNLOAD_DELAY
 
 Default: ``0``
 
-The amount of time (in secs) that the downloader should wait before downloading
-consecutive pages from the same website. This can be used to throttle the
-crawling speed to avoid hitting servers too hard. Decimal numbers are
-supported.  Example::
+Minimum seconds to wait between 2 consecutive requests to the same domain.
 
-    DOWNLOAD_DELAY = 0.25    # 250 ms of delay
+Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
+servers too hard.
+
+Decimal numbers are supported. For example, to send a maximum of 4 requests
+every 10 seconds::
+
+    DOWNLOAD_DELAY = 2.5
 
 This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
-setting (which is enabled by default). By default, Scrapy doesn't wait a fixed
-amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`.
+setting, which is enabled by default.
 
 When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
-per ip address instead of per domain.
+per IP address instead of per domain.
+
+Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
+concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
+time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
+concurrency for that domain is 1. When testing throttling configurations, it
+usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first,
+and only increase :setting:`DOWNLOAD_DELAY` once
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is
+desired.
 
 .. _spider-download_delay-attribute:
 
@@ -656,6 +667,11 @@ per ip address instead of per domain.
 
     This delay can be set per spider using :attr:`download_delay` spider attribute.
 
+It is also possible to change this setting per domain, although it requires
+non-trivial code. See the implementation of the :ref:`AutoThrottle
+<topics-autothrottle>` extension for an example.
+
+
 .. setting:: DOWNLOAD_HANDLERS
 
 DOWNLOAD_HANDLERS
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index ffe41cf3e..7aa8555d5 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -99,7 +99,7 @@ scrapy.Spider
    .. attribute:: crawler
 
       This attribute is set by the :meth:`from_crawler` class method after
-      initializating the class, and links to the
+      initializing the class, and links to the
       :class:`~scrapy.crawler.Crawler` object to which this spider instance is
       bound.
 
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 83bb08561..bb8527af2 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -1,9 +1,10 @@
 """
-A spider that generate light requests to meassure QPS throughput
+A spider that generate light requests to measure QPS throughput
 
 usage:
 
-    scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
+    scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0
+     --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
 
 """
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 1d74923a9..ffb40e1c5 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -24,7 +24,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
 
 
 def _iter_command_classes(module_name):
-    # TODO: add `name` attribute to commands and and merge this function with
+    # TODO: add `name` attribute to commands and merge this function with
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
         for obj in vars(module).values():
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 9dd2df638..1513638df 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -83,7 +83,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
     def getContext(self, hostname=None, port=None):
-        return self.getCertificateOptions().getContext()
+        ctx = self.getCertificateOptions().getContext()
+        ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
+        return ctx
 
     def creatorForNetloc(self, hostname, port):
         return ScrapyClientTLSOptions(
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index db2639a75..d1c511db0 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -23,8 +23,8 @@ METHOD_TLSv12 = "TLSv1.2"
 openssl_methods = {
     METHOD_TLS: SSL.SSLv23_METHOD,  # protocol negotiation (recommended)
     METHOD_TLSv10: SSL.TLSv1_METHOD,  # TLS 1.0 only
-    METHOD_TLSv11: getattr(SSL, "TLSv1_1_METHOD", 5),  # TLS 1.1 only
-    METHOD_TLSv12: getattr(SSL, "TLSv1_2_METHOD", 6),  # TLS 1.2 only
+    METHOD_TLSv11: SSL.TLSv1_1_METHOD,  # TLS 1.1 only
+    METHOD_TLSv12: SSL.TLSv1_2_METHOD,  # TLS 1.2 only
 }
 
 
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index a00afb7af..6421391d0 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -101,7 +101,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 # This class used to inherit from Twisted’s
 # twisted.web.client.HTTPClientFactory. When that class was deprecated in
 # Twisted (https://github.com/twisted/twisted/pull/643), we merged its
-# non-overriden code into this class.
+# non-overridden code into this class.
 class ScrapyHTTPClientFactory(ClientFactory):
 
     protocol = ScrapyHTTPPageGetter
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 1b185cd8c..2b5c98e5f 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -348,7 +348,7 @@ class Stream:
 
     def receive_headers(self, headers: List[HeaderTuple]) -> None:
         for name, value in headers:
-            self._response["headers"][name] = value
+            self._response["headers"].appendlist(name, value)
 
         # Check if we exceed the allowed max data size which can be received
         expected_size = int(self._response["headers"].get(b"Content-Length", -1))
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ca4d77a35..823955aa3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -384,11 +384,11 @@ class FeedExporter:
         return defer.DeferredList(deferred_list) if deferred_list else None
 
     def _close_slot(self, slot, spider):
+        slot.finish_exporting()
         if not slot.itemcount and not slot.store_empty:
             # We need to call slot.storage.store nonetheless to get the file
             # properly closed.
             return defer.maybeDeferred(slot.storage.store, slot.file)
-        slot.finish_exporting()
         logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
         d = defer.maybeDeferred(slot.storage.store, slot.file)
 
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 3bfbddc54..bbddaac40 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -196,7 +196,7 @@ class RFC2616Policy:
         if response.status in (300, 301, 308):
             return self.MAXAGE
 
-        # Insufficient information to compute fresshness lifetime
+        # Insufficient information to compute freshness lifetime
         return 0
 
     def _compute_current_age(self, response, request, now):
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ea519fb19..993219745 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -200,7 +200,7 @@ def _select_value(ele: SelectElement, n: str, v: str):
         o = ele.value_options
         return (n, o[0]) if o else (None, None)
     if v is not None and multiple:
-        # This is a workround to bug in lxml fixed 2.3.1
+        # This is a workaround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
         selected_options = ele.xpath(".//option[@selected]")
         values = [(o.get("value") or o.text or "").strip() for o in selected_options]
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 0bd28a882..66cf58d89 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -226,7 +226,8 @@ class LxmlLinkExtractor:
         Only links that match the settings passed to the ``__init__`` method of
         the link extractor are returned.
 
-        Duplicate links are omitted.
+        Duplicate links are omitted if the ``unique`` attribute is set to ``True``,
+        otherwise they are returned.
         """
         base_url = get_base_url(response)
         if self.restrict_xpaths:
@@ -239,4 +240,6 @@ class LxmlLinkExtractor:
         for doc in docs:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
-        return unique_list(all_links)
+        if self.link_extractor.unique:
+            return unique_list(all_links)
+        return all_links
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index fcecb5fbb..0cfa5665a 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -151,8 +151,8 @@ class ImagesPipeline(FilesPipeline):
             )
             if self._deprecated_convert_image:
                 warnings.warn(
-                    f"{self.__class__.__name__}.convert_image() method overriden in a deprecated way, "
-                    "overriden method does not accept response_body argument.",
+                    f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, "
+                    "overridden method does not accept response_body argument.",
                     category=ScrapyDeprecationWarning,
                 )
 
diff --git a/scrapy/shell.py b/scrapy/shell.py
index a42dedc93..084a27141 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -177,7 +177,11 @@ class Shell:
 
 def inspect_response(response, spider):
     """Open a shell to inspect the given response"""
+    # Shell.start removes the SIGINT handler, so save it and re-add it after
+    # the shell has closed
+    sigint_handler = signal.getsignal(signal.SIGINT)
     Shell(spider.crawler).start(response=response, spider=spider)
+    signal.signal(signal.SIGINT, sigint_handler)
 
 
 def _request_deferred(request):
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index b64098d7b..100f040bb 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,5 +1,4 @@
 from functools import wraps
-from collections import OrderedDict
 
 
 def _embed_ipython_shell(namespace={}, banner=""):
@@ -70,14 +69,12 @@ def _embed_standard_shell(namespace={}, banner=""):
     return wrapper
 
 
-DEFAULT_PYTHON_SHELLS = OrderedDict(
-    [
-        ("ptpython", _embed_ptpython_shell),
-        ("ipython", _embed_ipython_shell),
-        ("bpython", _embed_bpython_shell),
-        ("python", _embed_standard_shell),
-    ]
-)
+DEFAULT_PYTHON_SHELLS = {
+    "ptpython": _embed_ptpython_shell,
+    "ipython": _embed_ipython_shell,
+    "bpython": _embed_bpython_shell,
+    "python": _embed_standard_shell,
+}
 
 
 def get_shell_embed_func(shells=None, known_shells=None):
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 4af2a647a..21cd5e78f 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -26,10 +26,7 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import (
-    is_asyncio_reactor_installed,
-    get_asyncio_event_loop_policy,
-)
+from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -290,7 +287,7 @@ def deferred_from_coro(o) -> Any:
             # that use asyncio, e.g. "await asyncio.sleep(1)"
             return ensureDeferred(o)
         # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-        event_loop = get_asyncio_event_loop_policy().get_event_loop()
+        event_loop = _get_asyncio_event_loop()
         return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
     return o
 
@@ -343,8 +340,7 @@ def deferred_to_future(d: Deferred) -> Future:
                 d = treq.get('https://example.com/additional')
                 additional_response = await deferred_to_future(d)
     """
-    policy = get_asyncio_event_loop_policy()
-    return d.asFuture(policy.get_event_loop())
+    return d.asFuture(_get_asyncio_event_loop())
 
 
 def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index e40016031..c20948fd3 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,6 +1,7 @@
 import asyncio
 import sys
 from contextlib import suppress
+from warnings import catch_warnings, filterwarnings
 
 from twisted.internet import asyncioreactor, error
 
@@ -83,6 +84,10 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
+def _get_asyncio_event_loop():
+    return set_asyncio_event_loop(None)
+
+
 def set_asyncio_event_loop(event_loop_path):
     """Sets and returns the event loop with specified import path."""
     policy = get_asyncio_event_loop_policy()
@@ -92,11 +97,26 @@ def set_asyncio_event_loop(event_loop_path):
         asyncio.set_event_loop(event_loop)
     else:
         try:
-            event_loop = policy.get_event_loop()
+            with catch_warnings():
+                # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning
+                # is emitted about the lack of a current event loop, because in
+                # Python 3.14 and later `get_event_loop` will raise a
+                # RuntimeError in that event. Because our code is already
+                # prepared for that future behavior, we ignore the deprecation
+                # warning.
+                filterwarnings(
+                    "ignore",
+                    message="There is no current event loop",
+                    category=DeprecationWarning,
+                )
+                event_loop = policy.get_event_loop()
         except RuntimeError:
-            # `get_event_loop` is expected to fail when called from a new thread
-            # with no asyncio event loop yet installed. Such is the case when
-            # called from `scrapy shell`
+            # `get_event_loop` raises RuntimeError when called with no asyncio
+            # event loop yet installed in the following scenarios:
+            # - From a thread other than the main thread. For example, when
+            #   using ``scrapy shell``.
+            # - Previsibly on Python 3.14 and later.
+            #   https://github.com/python/cpython/issues/100160#issuecomment-1345581902
             event_loop = policy.new_event_loop()
             asyncio.set_event_loop(event_loop)
     return event_loop
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 3693177e4..a91a49170 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -40,7 +40,7 @@ def get_meta_refresh(
     response: "scrapy.http.response.text.TextResponse",
     ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
-    """Parse the http-equiv refrsh parameter from the given response"""
+    """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
         _metaref_cache[response] = html.get_meta_refresh(
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index b73261723..9f03621c1 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,14 +1,9 @@
-import OpenSSL
+import OpenSSL.SSL
 import OpenSSL._util as pyOpenSSLutil
 
 from scrapy.utils.python import to_unicode
 
 
-# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
-# Using the binding directly, as this code does, requires cryptography 2.4.
-SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, "SSL_OP_NO_TLSv1_3", 0)
-
-
 def ffi_buf_to_string(buf):
     return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
@@ -24,11 +19,6 @@ def x509name_to_string(x509name):
 
 
 def get_temp_key_info(ssl_object):
-    if not hasattr(
-        pyOpenSSLutil.lib, "SSL_get_server_tmp_key"
-    ):  # requires OpenSSL 1.0.2
-        return None
-
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
     temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
     if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index ad3bff4f0..cd8a6a05a 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -48,7 +48,7 @@ def parse_url(url, encoding=None):
 
 def escape_ajax(url):
     """
-    Return the crawleable url according to:
+    Return the crawlable url according to:
     https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
 
     >>> escape_ajax("www.example.com/ajax.html#!key=value")
diff --git a/sep/sep-016.rst b/sep/sep-016.rst
index 335f09f45..a60ab30dd 100644
--- a/sep/sep-016.rst
+++ b/sep/sep-016.rst
@@ -148,7 +148,7 @@ Another example could be for building URL canonicalizers:
 ::
 
    #!python
-   class CanonializeUrl(LegSpider):
+   class CanonicalizeUrl(LegSpider):
 
        def process_request(self, request):
            curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index fe707923a..96df82f6b 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -321,7 +321,7 @@ Another example could be for building URL canonicalizers:
 ::
 
    #!python
-   class CanonializeUrl(object):
+   class CanonicalizeUrl(object):
 
        def process_request(self, request, response, spider):
            curl = canonicalize_url(request.url, 
@@ -594,18 +594,18 @@ A middleware to Scrape data using Parsley as described in UsingParsley
 
    class ParsleyExtractor(object):
 
-       def __init__(self, parslet_json_code):
-           parslet = json.loads(parselet_json_code)
+       def __init__(self, parsley_json_code):
+           parsley = json.loads(parselet_json_code)
            class ParsleyItem(Item):
                def __init__(self, *a, **kw):
-                   for name in parslet.keys():
+                   for name in parsley.keys():
                        self.fields[name] = Field()
                super(ParsleyItem, self).__init__(*a, **kw)
            self.item_class = ParsleyItem    
-           self.parsley = PyParsley(parslet, output='python') 
+           self.parsley = PyParsley(parsley, output='python') 
 
        def process_response(self, response, request, spider):
-           return self.item_class(self.parsly.parse(string=response.body))
+           return self.item_class(self.parsley.parse(string=response.body))
 
 
 
diff --git a/sep/sep-021.rst b/sep/sep-021.rst
index c1ec16f7f..d56bc26af 100644
--- a/sep/sep-021.rst
+++ b/sep/sep-021.rst
@@ -79,7 +79,7 @@ If it raises an exception, Scrapy will print it and exit.
 Examples::
 
     def addon_configure(settings):
-        settings.overrides['DOWNLADER_MIDDLEWARES'].update({
+        settings.overrides['DOWNLOADER_MIDDLEWARES'].update({
             'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
         })
 
diff --git a/setup.py b/setup.py
index 5d1245f37..f53334d4e 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     "Twisted>=18.9.0",
-    "cryptography>=3.3",
+    "cryptography>=3.4.6",
     "cssselect>=0.9.1",
     "itemloaders>=1.0.1",
     "parsel>=1.5.0",
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 9f1a707c9..e07ae8797 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -19,7 +19,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
 from scrapy.utils.test import get_testenv
 
 
@@ -358,7 +357,7 @@ def ssl_context_factory(
     if cipher_string:
         ctx = factory.getContext()
         # disabling TLS1.3 because it unconditionally enables some strong ciphers
-        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3)
+        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3)
         ctx.set_cipher_list(to_bytes(cipher_string))
     return factory
 
diff --git a/tests/pipelines.py b/tests/pipelines.py
index 4e73178d5..d9fc12676 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -11,6 +11,6 @@ class ZeroDivisionErrorPipeline:
         return item
 
 
-class ProcessWithZeroDivisionErrorPipiline:
+class ProcessWithZeroDivisionErrorPipeline:
     def process_item(self, item, spider):
         1 / 0
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index e3a2a4145..29075602d 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -13,6 +13,7 @@
       </div>
       <a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
       <a href='sample3.html'>sample 3 repetition</a>
+      <a href='sample3.html'>sample 3 repetition</a>
       <a href='sample3.html#foo'>sample 3 repetition with fragment</a>
       <a href='http://www.google.com/something'></a>
       <a href='http://example.com/innertag.html'><strong>inner</strong> tag</a>
diff --git a/tests/test_commands.py b/tests/test_commands.py
index ed0eb85f5..363e87aa7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -336,7 +336,7 @@ class StartprojectTemplatesTest(ProjectTest):
         self.assertEqual(actual_permissions, expected_permissions)
 
     def test_startproject_permissions_unchanged_in_destination(self):
-        """Check that pre-existing folders and files in the destination folder
+        """Check that preexisting folders and files in the destination folder
         do not see their permissions modified."""
         scrapy_path = scrapy.__path__[0]
         project_template = Path(scrapy_path, "templates", "project")
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3f19bef22..4139f1b11 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -154,7 +154,7 @@ class CrawlTestCase(TestCase):
             raise unittest.SkipTest("Non-existing hosts are resolvable")
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            # try to fetch the homepage of a non-existent domain
+            # try to fetch the homepage of a nonexistent domain
             yield crawler.crawl(
                 "http://dns.resolution.invalid./", mockserver=self.mockserver
             )
@@ -183,7 +183,7 @@ class CrawlTestCase(TestCase):
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
-    def test_start_requests_lazyness(self):
+    def test_start_requests_laziness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = get_crawler(BrokenStartRequestsSpider, settings)
         yield crawler.crawl(mockserver=self.mockserver)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 536d09f18..4f953439d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -209,6 +209,12 @@ class LargeChunkedFileResource(resource.Resource):
         return server.NOT_DONE_YET
 
 
+class DuplicateHeaderResource(resource.Resource):
+    def render(self, request):
+        request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
+        return b""
+
+
 class HttpTestCase(unittest.TestCase):
     scheme = "http"
     download_handler_cls: Type = HTTPDownloadHandler
@@ -234,6 +240,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"contentlength", ContentLengthHeaderResource())
         r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
         r.putChild(b"largechunkedfile", LargeChunkedFileResource())
+        r.putChild(b"duplicate-header", DuplicateHeaderResource())
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
@@ -407,6 +414,16 @@ class HttpTestCase(unittest.TestCase):
             HtmlResponse,
         )
 
+    def test_get_duplicate_header(self):
+        def _test(response):
+            self.assertEqual(
+                response.headers.getlist(b"Set-Cookie"),
+                [b"a=b", b"c=d"],
+            )
+
+        request = Request(self.getURL("duplicate-header"))
+        return self.download_request(request, Spider("foo")).addCallback(_test)
+
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
@@ -1095,9 +1112,9 @@ class BaseFTPTestCase(unittest.TestCase):
 
         return self._add_test_callbacks(d, _test)
 
-    def test_ftp_download_notexist(self):
+    def test_ftp_download_nonexistent(self):
         request = Request(
-            url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta
+            url=f"ftp://127.0.0.1:{self.portNum}/nonexistent.txt", meta=self.req_meta
         )
         d = self.download_handler.download_request(request, None)
 
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 927f0ae4d..0702dd042 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -19,7 +19,7 @@ class UserAgentMiddlewareTest(TestCase):
         self.assertEqual(req.headers["User-Agent"], b"default_useragent")
 
     def test_remove_agent(self):
-        # settings UESR_AGENT to None should remove the user agent
+        # settings USER_AGENT to None should remove the user agent
         spider, mw = self.get_spider_and_mw("default_useragent")
         spider.user_agent = None
         mw.spider_opened(spider)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 7b861332e..7ddb420ba 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -109,7 +109,7 @@ class DataClassItemsSpider(TestSpider):
 class ItemZeroDivisionErrorSpider(TestSpider):
     custom_settings = {
         "ITEM_PIPELINES": {
-            "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300,
+            "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
         }
     }
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 890c88c7e..09a4aa823 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -33,8 +33,9 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.exporters import CsvItemExporter
+from scrapy.exporters import CsvItemExporter, JsonItemExporter
 from scrapy.extensions.feedexport import (
+    _FeedSlot,
     BlockingFeedStorage,
     FeedExporter,
     FileFeedStorage,
@@ -664,6 +665,50 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return result
 
 
+class InstrumentedFeedSlot(_FeedSlot):
+    """Instrumented _FeedSlot subclass for keeping track of calls to
+    start_exporting and finish_exporting."""
+
+    def start_exporting(self):
+        self.update_listener("start")
+        super().start_exporting()
+
+    def finish_exporting(self):
+        self.update_listener("finish")
+        super().finish_exporting()
+
+    @classmethod
+    def subscribe__listener(cls, listener):
+        cls.update_listener = listener.update
+
+
+class IsExportingListener:
+    """When subscribed to InstrumentedFeedSlot, keeps track of when
+    a call to start_exporting has been made without a closing call to
+    finish_exporting and when a call to finish_exporting has been made
+    before a call to start_exporting."""
+
+    def __init__(self):
+        self.start_without_finish = False
+        self.finish_without_start = False
+
+    def update(self, method):
+        if method == "start":
+            self.start_without_finish = True
+        elif method == "finish":
+            if self.start_without_finish:
+                self.start_without_finish = False
+            else:
+                self.finish_before_start = True
+
+
+class ExceptionJsonItemExporter(JsonItemExporter):
+    """JsonItemExporter that throws an exception every time export_item is called."""
+
+    def export_item(self, _):
+        raise Exception("foo")
+
+
 class FeedExportTest(FeedExportTestBase):
     __test__ = True
 
@@ -909,6 +954,84 @@ class FeedExportTest(FeedExportTestBase):
             data = yield self.exported_no_data(settings)
             self.assertEqual(b"", data[fmt])
 
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_items(self):
+        items = [
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+        ]
+        settings = {
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+            },
+            "FEED_EXPORT_INDENT": None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
+
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_no_items(self):
+        items = []
+        settings = {
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+            },
+            "FEED_EXPORT_INDENT": None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
+
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_items_exception(self):
+        items = [
+            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+        ]
+        settings = {
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+            },
+            "FEED_EXPORTERS": {"json": ExceptionJsonItemExporter},
+            "FEED_EXPORT_INDENT": None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
+
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_no_items_exception(self):
+        items = []
+        settings = {
+            "FEEDS": {
+                self._random_temp_filename(): {"format": "json"},
+            },
+            "FEED_EXPORTERS": {"json": ExceptionJsonItemExporter},
+            "FEED_EXPORT_INDENT": None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
+
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
         formats = (
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index b6f999b29..0c10b27a0 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -399,7 +399,7 @@ class RequestTest(unittest.TestCase):
             )
             self.assertEqual(r.method, "DELETE")
 
-        # If `ignore_unknon_options` is set to `False` it raises an error with
+        # If `ignore_unknown_options` is set to `False` it raises an error with
         # the unknown options: --foo and -z
         self.assertRaises(
             ValueError,
@@ -997,7 +997,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]})
 
-    def test_from_response_formname_notexist(self):
+    def test_from_response_formname_nonexistent(self):
         response = _buildresponse(
             """<form name="form1" action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
@@ -1044,7 +1044,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]})
 
-    def test_from_response_formname_notexists_fallback_formid(self):
+    def test_from_response_formname_nonexistent_fallback_formid(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
@@ -1062,7 +1062,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]})
 
-    def test_from_response_formid_notexist(self):
+    def test_from_response_formid_nonexistent(self):
         response = _buildresponse(
             """<form id="form1" action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index ce32092c1..8c422bb4f 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -518,7 +518,7 @@ class TextResponseTest(BaseResponseTest):
     def test_bom_is_removed_from_body(self):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
-        # response.text in indistint order doesn't affect final
+        # response.text in indistinct order doesn't affect final
         # values for encoding and decoded body.
         url = "http://example.com"
         body = b"\xef\xbb\xbfWORD"
@@ -645,6 +645,7 @@ class TextResponseTest(BaseResponseTest):
             "http://example.com/sample2.html",
             "http://example.com/sample3.html",
             "http://example.com/sample3.html",
+            "http://example.com/sample3.html",
             "http://example.com/sample3.html#foo",
             "http://www.google.com/something",
             "http://example.com/innertag.html",
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 520effedb..f663013ba 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -74,6 +74,10 @@ class Base:
                         url="http://example.com/sample3.html",
                         text="sample 3 repetition",
                     ),
+                    Link(
+                        url="http://example.com/sample3.html",
+                        text="sample 3 repetition",
+                    ),
                     Link(
                         url="http://example.com/sample3.html#foo",
                         text="sample 3 repetition with fragment",
@@ -93,6 +97,10 @@ class Base:
                         url="http://example.com/sample3.html",
                         text="sample 3 repetition",
                     ),
+                    Link(
+                        url="http://example.com/sample3.html",
+                        text="sample 3 repetition",
+                    ),
                     Link(
                         url="http://example.com/sample3.html",
                         text="sample 3 repetition with fragment",
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 3cd3e5966..38a2d6c41 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -225,8 +225,8 @@ class ImagesPipelineTestCase(unittest.TestCase):
                 self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
 
                 expected_warning_msg = (
-                    ".convert_image() method overriden in a deprecated way, "
-                    "overriden method does not accept response_body argument."
+                    ".convert_image() method overridden in a deprecated way, "
+                    "overridden method does not accept response_body argument."
                 )
                 self.assertEqual(
                     len(
@@ -244,7 +244,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter("always")
             SIZE = (100, 100)
-            # straigh forward case: RGB and JPEG
+            # straight forward case: RGB and JPEG
             COLOUR = (0, 127, 255)
             im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)
             converted, _ = self.pipeline.convert_image(im)
@@ -271,7 +271,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
             self.assertEqual(converted.mode, "RGB")
             self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
-            # ensure that we recieved deprecation warnings
+            # ensure that we received deprecation warnings
             expected_warning_msg = ".convert_image() method called in a deprecated way"
             self.assertTrue(
                 len(
@@ -287,7 +287,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
     def test_convert_image_new(self):
         # tests for new API
         SIZE = (100, 100)
-        # straigh forward case: RGB and JPEG
+        # straight forward case: RGB and JPEG
         COLOUR = (0, 127, 255)
         im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)
         converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index f7540743d..17c0309d1 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -11,12 +11,12 @@ from tests.mockserver import MockServer
 from tests.spiders import SingleRequestSpider
 
 
-OVERRIDEN_URL = "https://example.org"
+OVERRIDDEN_URL = "https://example.org"
 
 
 class ProcessResponseMiddleware:
     def process_response(self, request, response, spider):
-        return response.replace(request=Request(OVERRIDEN_URL))
+        return response.replace(request=Request(OVERRIDDEN_URL))
 
 
 class RaiseExceptionRequestMiddleware:
@@ -30,7 +30,7 @@ class CatchExceptionOverrideRequestMiddleware:
         return Response(
             url="http://localhost/",
             body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
-            request=Request(OVERRIDEN_URL),
+            request=Request(OVERRIDDEN_URL),
         )
 
 
@@ -52,7 +52,7 @@ class AlternativeCallbacksSpider(SingleRequestSpider):
 class AlternativeCallbacksMiddleware:
     def process_response(self, request, response, spider):
         new_request = request.replace(
-            url=OVERRIDEN_URL,
+            url=OVERRIDDEN_URL,
             callback=spider.alt_callback,
             cb_kwargs={"foo": "bar"},
         )
@@ -132,16 +132,16 @@ class CrawlTestCase(TestCase):
             yield crawler.crawl(seed=url, mockserver=self.mockserver)
 
         response = crawler.spider.meta["responses"][0]
-        self.assertEqual(response.request.url, OVERRIDEN_URL)
+        self.assertEqual(response.request.url, OVERRIDDEN_URL)
 
         self.assertEqual(signal_params["response"].url, url)
-        self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
+        self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL)
 
         log.check_present(
             (
                 "scrapy.core.engine",
                 "DEBUG",
-                f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)",
+                f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)",
             ),
         )
 
@@ -166,7 +166,7 @@ class CrawlTestCase(TestCase):
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
-        self.assertEqual(response.request.url, OVERRIDEN_URL)
+        self.assertEqual(response.request.url, OVERRIDDEN_URL)
 
     @defer.inlineCallbacks
     def test_downloader_middleware_do_not_override_in_process_exception(self):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index db5a22adb..dad39b6ee 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -227,7 +227,7 @@ class MixinSameOrigin:
         ),
         ("http://example.com:81/page.html", "http://example.com/not-page.html", None),
         ("http://example.com/page.html", "http://example.com:81/not-page.html", None),
-        # Different protocols: do NOT send refferer
+        # Different protocols: do NOT send referrer
         ("https://example.com/page.html", "http://example.com/not-page.html", None),
         ("https://example.com/page.html", "http://not.example.com/", None),
         ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
@@ -750,19 +750,19 @@ class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
 
 
-class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
     req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
 
 
-class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
+class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {
         "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
     }
     req_meta = {"referrer_policy": POLICY_NO_REFERRER}
 
 
-class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {
         "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
     }
@@ -888,19 +888,19 @@ class TestSettingsPolicyByName(TestCase):
             RefererMiddleware(settings)
 
 
-class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
     resp_headers = {"Referrer-Policy": POLICY_UNSAFE_URL.upper()}
 
 
-class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {
         "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
     }
     resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER.swapcase()}
 
 
-class TestPolicyHeaderPredecence003(
+class TestPolicyHeaderPrecedence003(
     MixinNoReferrerWhenDowngrade, TestRefererMiddleware
 ):
     settings = {
@@ -909,7 +909,7 @@ class TestPolicyHeaderPredecence003(
     resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
 
-class TestPolicyHeaderPredecence004(
+class TestPolicyHeaderPrecedence004(
     MixinNoReferrerWhenDowngrade, TestRefererMiddleware
 ):
     """
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 25cef2b26..d09335651 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -15,6 +15,11 @@ class AsyncioTest(TestCase):
         )
 
     def test_install_asyncio_reactor(self):
+        from twisted.internet import reactor as original_reactor
+
         with warnings.catch_warnings(record=True) as w:
             install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
             self.assertEqual(len(w), 0)
+        from twisted.internet import reactor
+
+        assert original_reactor == reactor
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 1a8c96e7a..214deceb2 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -74,7 +74,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         self.assertIn("foo.NewClass", str(w[1].message))
         self.assertIn("bar.OldClass", str(w[1].message))
 
-    def test_subclassing_warns_only_on_direct_childs(self):
+    def test_subclassing_warns_only_on_direct_children(self):
         Deprecated = create_deprecated_class(
             "Deprecated", NewName, warn_once=False, warn_category=MyWarning
         )
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 43236acec..da61f4b0b 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -7,17 +7,27 @@ from scrapy.utils.display import pformat, pprint
 
 class TestDisplay(TestCase):
     object = {"a": 1}
-    colorized_string = (
-        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
-        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
-    )
+    colorized_strings = {
+        (
+            (
+                "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+                "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
+            )
+            + suffix
+        )
+        for suffix in (
+            # https://github.com/pygments/pygments/issues/2313
+            "\n",  # pygments ≤ 2.13
+            "\x1b[37m\x1b[39;49;00m\n",  # pygments ≥ 2.14
+        )
+    }
     plain_string = "{'a': 1}"
 
     @mock.patch("sys.platform", "linux")
     @mock.patch("sys.stdout.isatty")
     def test_pformat(self, isatty):
         isatty.return_value = True
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch("sys.stdout.isatty")
     def test_pformat_dont_colorize(self, isatty):
@@ -33,7 +43,7 @@ class TestDisplay(TestCase):
     def test_pformat_old_windows(self, isatty, version):
         isatty.return_value = True
         version.return_value = "10.0.14392"
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch("sys.platform", "win32")
     @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@@ -55,7 +65,7 @@ class TestDisplay(TestCase):
         isatty.return_value = True
         version.return_value = "10.0.14393"
         terminal_processing.return_value = True
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch("sys.platform", "linux")
     @mock.patch("sys.stdout.isatty")
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3c6270864..5caa5b8f2 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -159,7 +159,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         b = Obj()
         # no attributes given return False
         self.assertFalse(equal_attributes(a, b, []))
-        # not existent attributes
+        # nonexistent attributes
         self.assertFalse(equal_attributes(a, b, ["x", "y"]))
 
         a.x = 1
diff --git a/tox.ini b/tox.ini
index e37ae4579..eaf29173a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,7 +32,7 @@ download = true
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =
-    pip install -U -ctests/upper-constraints.txt {opts} {packages}
+    python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
 
 [testenv:typing]
 basepython = python3
@@ -63,8 +63,7 @@ commands =
     flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
-# reppy does not support Python 3.9+
-basepython = python3.8
+basepython = python3
 deps =
     {[testenv:extra-deps]deps}
     pylint==2.15.6
@@ -75,13 +74,14 @@ commands =
 basepython = python3
 deps =
     twine==4.0.1
+    build==0.9.0
 commands =
-    python setup.py sdist
+    python -m build --sdist
     twine check dist/*
 
 [pinned]
 deps =
-    cryptography==3.3
+    cryptography==3.4.6
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
@@ -106,7 +106,7 @@ deps =
 setenv =
     _SCRAPY_PINNED=true
 install_command =
-    pip install -U {opts} {packages}
+    python -I -m pip install {opts} {packages}
 
 [testenv:pinned]
 deps =
@@ -126,8 +126,7 @@ setenv =
     {[pinned]setenv}
 
 [testenv:extra-deps]
-# reppy does not support Python 3.9+
-basepython = python3.8
+basepython = python3
 deps =
     {[testenv]deps}
     boto
@@ -135,7 +134,6 @@ deps =
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
-    reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
     Twisted[http2]>=17.9.0

From 72a853c751044172d90704bce1adb6780e50934e Mon Sep 17 00:00:00 2001
From: Emmanuel Rondan <emarondan@pop-os.localdomain>
Date: Mon, 23 Jan 2023 14:56:48 -0300
Subject: [PATCH 15/15] fixing contributing page with suggestions

---
 docs/contributing.rst | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 12f0e8d21..edc6c2179 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -179,27 +179,30 @@ Scrapy:
   See https://help.github.com/en/github/using-git/setting-your-username-in-git for
   setup instructions.
 
-.. _pre-commit:
-
 Pre-commit
 ==========
 
 We use `pre-commit`_ to automatically address simple code issues before every 
 commit.
 
+.. _pre-commit: https://pre-commit.com/
+
 Before you start writing a patch:
-  #.  `Install pre-commit <https://pre-commit.com/#installation>`_.
-  #.  On the root of your local clone of the Scrapy repository, run the following command:
+
+#.  `Install pre-commit <https://pre-commit.com/#installation>`_.
+
+#.  On the root of your local clone of the Scrapy repository, run the following 
+    command:
 
     .. code-block:: bash
-      
-      pre-commit install
 
-Now our pre-commit hooks will run every time you create a Git commit. Upon 
-finding issues, pre-commit hooks aborts your commit, and they either fix 
-the corresponding issues automatically or only report them to you. If they fix 
-the issues automatically, creating your commit again should succeed. Otherwise, 
-you may need to address the corresponding issues manually first.
+       pre-commit install
+
+Now pre-commit will check your changes every time you create a Git commit. Upon 
+finding issues, pre-commit aborts your commit, and either fixes those issues 
+automatically, or only reports them to you. If it fixes those issues 
+automatically, creating your commit again should succeed. Otherwise, you may 
+need to address the corresponding issues manually first.
 
 .. _documentation-policies: