Upgrade CI tools

.pre-commit-config.yaml

@@ -5,14 +5,14 @@ repos:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 6.0.0
+  rev: 5.0.4  # 6.0.0 drops Python 3.7 support
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
-  rev: 22.12.0
+  rev: 23.1.0
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.12.0
+  rev: 5.11.5  # 5.12 drops Python 3.7 support
   hooks:
   - id: isort

docs/utils/linkfix.py

@@ -18,7 +18,6 @@ from pathlib import Path
 
 
 def main():
-
     # Used for remembering the file (and its contents)
     # so we don't have to open the same file again.
     _filename = None
@@ -50,7 +49,6 @@ def main():
             else:
                 # If this is a new file
                 if newfilename != _filename:
-
                     # Update the previous file
                     if _filename:
                         Path(_filename).write_text(_contents, encoding="utf-8")

extras/qpsclient.py

@@ -13,7 +13,6 @@ from scrapy.spiders import Spider
 
 
 class QPSSpider(Spider):
-
     name = "qps"
     benchurl = "http://localhost:8880/"
 

scrapy/commands/__init__.py

@@ -14,7 +14,6 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 class ScrapyCommand:
-
     requires_project = False
     crawler_process: Optional[CrawlerProcess] = None
 

scrapy/commands/bench.py

@@ -9,7 +9,6 @@ from scrapy.linkextractors import LinkExtractor
 
 
 class Command(ScrapyCommand):
-
     default_settings = {
         "LOG_LEVEL": "INFO",
         "LOGSTATS_INTERVAL": 1,

scrapy/commands/crawl.py

@@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError
 
 
 class Command(BaseRunSpiderCommand):
-
     requires_project = True
 
     def syntax(self):

scrapy/commands/edit.py

@@ -6,7 +6,6 @@ from scrapy.exceptions import UsageError
 
 
 class Command(ScrapyCommand):
-
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 

scrapy/commands/fetch.py

@@ -10,7 +10,6 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
 
 class Command(ScrapyCommand):
-
     requires_project = False
 
     def syntax(self):

scrapy/commands/genspider.py

@@ -32,7 +32,6 @@ def extract_domain(url):
 
 
 class Command(ScrapyCommand):
-
     requires_project = False
     default_settings = {"LOG_ENABLED": False}
 

scrapy/commands/list.py

@@ -2,7 +2,6 @@ from scrapy.commands import ScrapyCommand
 
 
 class Command(ScrapyCommand):
-
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 

scrapy/commands/runspider.py

@@ -24,7 +24,6 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
 
 
 class Command(BaseRunSpiderCommand):
-
     requires_project = False
     default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
 

scrapy/commands/settings.py

@@ -5,7 +5,6 @@ from scrapy.settings import BaseSettings
 
 
 class Command(ScrapyCommand):
-
     requires_project = False
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 

scrapy/commands/shell.py

@@ -13,7 +13,6 @@ from scrapy.utils.url import guess_scheme
 
 
 class Command(ScrapyCommand):
-
     requires_project = False
     default_settings = {
         "KEEP_ALIVE": True,

scrapy/commands/startproject.py

@@ -28,7 +28,6 @@ def _make_writable(path):
 
 
 class Command(ScrapyCommand):
-
     requires_project = False
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 

scrapy/commands/version.py

@@ -4,7 +4,6 @@ from scrapy.utils.versions import scrapy_components_versions
 
 
 class Command(ScrapyCommand):
-
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
     def syntax(self):

scrapy/core/downloader/__init__.py

@@ -69,7 +69,6 @@ def _get_concurrency_delay(concurrency, spider, settings):
 
 
 class Downloader:
-
     DOWNLOAD_SLOT = "download_slot"
 
     def __init__(self, crawler):

scrapy/core/downloader/handlers/http11.py

@@ -292,7 +292,6 @@ class ScrapyProxyAgent(Agent):
 
 
 class ScrapyAgent:
-
     _Agent = Agent
     _ProxyAgent = ScrapyProxyAgent
     _TunnelingAgent = TunnelingAgent

scrapy/core/downloader/middleware.py

@@ -17,7 +17,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
 
 
 class DownloaderMiddlewareManager(MiddlewareManager):
-
     component_name = "downloader middleware"
 
     @classmethod

scrapy/core/downloader/webclient.py

@@ -40,7 +40,6 @@ def _parse(url):
 
 
 class ScrapyHTTPPageGetter(HTTPClient):
-
     delimiter = b"\n"
 
     def connectionMade(self):
@@ -103,7 +102,6 @@ class ScrapyHTTPPageGetter(HTTPClient):
 # Twisted (https://github.com/twisted/twisted/pull/643), we merged its
 # non-overridden code into this class.
 class ScrapyHTTPClientFactory(ClientFactory):
-
     protocol = ScrapyHTTPPageGetter
 
     waiting = 1

scrapy/core/spidermw.py

@@ -46,7 +46,6 @@ def _isiterable(o) -> bool:
 
 
 class SpiderMiddlewareManager(MiddlewareManager):
-
     component_name = "spider middleware"
 
     def __init__(self, *middlewares):

scrapy/downloadermiddlewares/ajaxcrawl.py

@@ -30,7 +30,6 @@ class AjaxCrawlMiddleware:
         return cls(crawler.settings)
 
     def process_response(self, request, response, spider):
-
         if not isinstance(response, HtmlResponse) or response.status != 200:
             return response
 

scrapy/downloadermiddlewares/httpcache.py

@@ -27,7 +27,6 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlew
 
 
 class HttpCacheMiddleware:
-
     DOWNLOAD_EXCEPTIONS = (
         defer.TimeoutError,
         TimeoutError,

scrapy/downloadermiddlewares/httpcompression.py

@@ -53,7 +53,6 @@ class HttpCompressionMiddleware:
         request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
-
         if request.method == "HEAD":
             return response
         if isinstance(response, Response):

scrapy/downloadermiddlewares/redirect.py

@@ -26,7 +26,6 @@ def _build_redirect_request(source_request, *, url, **kwargs):
 
 
 class BaseRedirectMiddleware:
-
     enabled_setting = "REDIRECT_ENABLED"
 
     def __init__(self, settings):
@@ -115,7 +114,6 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):
-
     enabled_setting = "METAREFRESH_ENABLED"
 
     def __init__(self, settings):

scrapy/downloadermiddlewares/retry.py

@@ -122,7 +122,6 @@ def get_retry_request(
 
 
 class RetryMiddleware:
-
     # IOError is raised by the HttpCompression middleware when trying to
     # decompress an empty response
     EXCEPTIONS_TO_RETRY = (

scrapy/extension.py

@@ -8,7 +8,6 @@ from scrapy.utils.conf import build_component_list
 
 
 class ExtensionManager(MiddlewareManager):
-
     component_name = "extension"
 
     @classmethod

scrapy/extensions/httpcache.py

@@ -41,7 +41,6 @@ class DummyPolicy:
 
 
 class RFC2616Policy:
-
     MAXAGE = 3600 * 24 * 365  # one year
 
     def __init__(self, settings):

scrapy/http/request/json_request.py

@@ -15,7 +15,6 @@ from scrapy.utils.deprecate import create_deprecated_class
 
 
 class JsonRequest(Request):
-
     attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
 
     def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:

scrapy/http/response/text.py

@@ -29,7 +29,6 @@ _NONE = object()
 
 
 class TextResponse(Response):
-
     _DEFAULT_ENCODING = "ascii"
     _cached_decoded_json = _NONE
 

scrapy/linkextractors/lxmlhtml.py

@@ -194,7 +194,6 @@ class LxmlLinkExtractor:
         return True
 
     def matches(self, url):
-
         if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
             return False
         if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):

scrapy/pipelines/__init__.py

@@ -10,7 +10,6 @@ from scrapy.utils.defer import deferred_f_from_coro_f
 
 
 class ItemPipelineManager(MiddlewareManager):
-
     component_name = "item pipeline"
 
     @classmethod

scrapy/pipelines/files.py

@@ -187,7 +187,6 @@ class S3FilesStore:
 
 
 class GCSFilesStore:
-
     GCS_PROJECT_ID = None
 
     CACHE_CONTROL = "max-age=172800"
@@ -253,7 +252,6 @@ class GCSFilesStore:
 
 
 class FTPFilesStore:
-
     FTP_USERNAME = None
     FTP_PASSWORD = None
     USE_ACTIVE_MODE = None

scrapy/pipelines/media.py

@@ -23,7 +23,6 @@ def _DUMMY_CALLBACK(response):
 
 
 class MediaPipeline:
-
     LOG_FAILED_RESULTS = True
 
     class SpiderInfo:

scrapy/responsetypes.py

@@ -12,7 +12,6 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
 class ResponseTypes:
-
     CLASSES = {
         "text/html": "scrapy.http.HtmlResponse",
         "application/atom+xml": "scrapy.http.XmlResponse",

scrapy/shell.py

@@ -25,7 +25,6 @@ from scrapy.utils.response import open_in_browser
 
 
 class Shell:
-
     relevant_classes = (Crawler, Spider, Request, Response, Settings)
 
     def __init__(self, crawler, update_vars=None, code=None):

scrapy/spidermiddlewares/referer.py

@@ -34,7 +34,6 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"
 
 
 class ReferrerPolicy:
-
     NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
     name: str
 

scrapy/spiders/crawl.py

@@ -60,7 +60,6 @@ class Rule:
 
 
 class CrawlSpider(Spider):
-
     rules: Sequence[Rule] = ()
 
     def __init__(self, *a, **kw):

scrapy/spiders/feed.py

@@ -89,7 +89,7 @@ class XMLFeedSpider(Spider):
             yield node
 
     def _register_namespaces(self, selector):
-        for (prefix, uri) in self.namespaces:
+        for prefix, uri in self.namespaces:
             selector.register_namespace(prefix, uri)
 
 

scrapy/spiders/sitemap.py

@@ -10,7 +10,6 @@ logger = logging.getLogger(__name__)
 
 
 class SitemapSpider(Spider):
-
     sitemap_urls = ()
     sitemap_rules = [("", "parse")]
     sitemap_follow = [""]

scrapy/utils/benchserver.py

@@ -6,7 +6,6 @@ from twisted.web.server import Site
 
 
 class Root(Resource):
-
     isLeaf = True
 
     def getChild(self, name, request):

scrapy/utils/datatypes.py

@@ -11,7 +11,6 @@ from collections.abc import Mapping
 
 
 class CaselessDict(dict):
-
     __slots__ = ()
 
     def __init__(self, seq=None):

scrapy/utils/deprecate.py

@@ -54,7 +54,6 @@ def create_deprecated_class(
     """
 
     class DeprecatedClass(new_class.__class__):
-
         deprecated_class = None
         warned_on_subclass = False
 

scrapy/utils/serialize.py

@@ -9,7 +9,6 @@ from scrapy.http import Request, Response
 
 
 class ScrapyJSONEncoder(json.JSONEncoder):
-
     DATE_FORMAT = "%Y-%m-%d"
     TIME_FORMAT = "%H:%M:%S"
 

scrapy/utils/testproc.py

@@ -5,7 +5,6 @@ from twisted.internet import defer, protocol
 
 
 class ProcessTest:
-
     command = None
     prefix = [sys.executable, "-m", "scrapy.cmdline"]
     cwd = os.getcwd()  # trial chdirs to temp dir

scrapy/utils/url.py

@@ -144,7 +144,6 @@ def strip_url(
     origin_only=False,
     strip_fragment=True,
 ):
-
     """Strip URL string from some of its components:
 
     - ``strip_credentials`` removes "user:password@"

tests/mockserver.py

@@ -95,7 +95,6 @@ class BrokenDownloadResource(resource.Resource):
 
 
 class LeafResource(resource.Resource):
-
     isLeaf = True
 
     def deferRequest(self, request, delay, f, *a, **kw):

tests/requirements.txt

@@ -2,7 +2,7 @@
 attrs
 pyftpdlib
 pytest
-pytest-cov==3.0.0
+pytest-cov==4.0.0
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures

tests/spiders.py

@@ -25,7 +25,6 @@ class MockServerSpider(Spider):
 
 
 class MetaSpider(MockServerSpider):
-
     name = "meta"
 
     def __init__(self, *args, **kwargs):
@@ -37,7 +36,6 @@ class MetaSpider(MockServerSpider):
 
 
 class FollowAllSpider(MetaSpider):
-
     name = "follow"
     link_extractor = LinkExtractor()
 
@@ -59,7 +57,6 @@ class FollowAllSpider(MetaSpider):
 
 
 class DelaySpider(MetaSpider):
-
     name = "delay"
 
     def __init__(self, n=1, b=0, *args, **kwargs):
@@ -81,7 +78,6 @@ class DelaySpider(MetaSpider):
 
 
 class SimpleSpider(MetaSpider):
-
     name = "simple"
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
@@ -93,7 +89,6 @@ class SimpleSpider(MetaSpider):
 
 
 class AsyncDefSpider(SimpleSpider):
-
     name = "asyncdef"
 
     async def parse(self, response):
@@ -102,7 +97,6 @@ class AsyncDefSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioSpider(SimpleSpider):
-
     name = "asyncdef_asyncio"
 
     async def parse(self, response):
@@ -112,7 +106,6 @@ class AsyncDefAsyncioSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioReturnSpider(SimpleSpider):
-
     name = "asyncdef_asyncio_return"
 
     async def parse(self, response):
@@ -123,7 +116,6 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
-
     name = "asyncdef_asyncio_return_single_element"
 
     async def parse(self, response):
@@ -134,7 +126,6 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
-
     name = "asyncdef_asyncio_reqs_return"
 
     async def parse(self, response):
@@ -191,7 +182,6 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioGenSpider(SimpleSpider):
-
     name = "asyncdef_asyncio_gen"
 
     async def parse(self, response):
@@ -201,7 +191,6 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
-
     name = "asyncdef_asyncio_gen_loop"
 
     async def parse(self, response):
@@ -212,7 +201,6 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
 
 
 class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
-
     name = "asyncdef_asyncio_gen_complex"
     initial_reqs = 4
     following_reqs = 3
@@ -246,7 +234,6 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
 
 
 class ItemSpider(FollowAllSpider):
-
     name = "item"
 
     def parse(self, response):
@@ -261,7 +248,6 @@ class DefaultError(Exception):
 
 
 class ErrorSpider(FollowAllSpider):
-
     name = "error"
     exception_cls = DefaultError
 
@@ -275,7 +261,6 @@ class ErrorSpider(FollowAllSpider):
 
 
 class BrokenStartRequestsSpider(FollowAllSpider):
-
     fail_before_yield = False
     fail_yielding = False
 
@@ -305,7 +290,6 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
 
 class SingleRequestSpider(MetaSpider):
-
     seed = None
     callback_func = None
     errback_func = None
@@ -451,7 +435,6 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse
 
 
 class BytesReceivedCallbackSpider(MetaSpider):
-
     full_response_length = 2**18
 
     @classmethod

tests/test_command_check.py

@@ -2,7 +2,6 @@ from tests.test_commands import CommandTest
 
 
 class CheckCommandTest(CommandTest):
-
     command = "check"
 
     def setUp(self):

tests/test_command_fetch.py

@@ -6,7 +6,6 @@ from scrapy.utils.testsite import SiteTest
 
 
 class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
-
     command = "fetch"
 
     @defer.inlineCallbacks

tests/test_command_shell.py

@@ -9,7 +9,6 @@ from tests import NON_EXISTING_RESOLVABLE, tests_datadir
 
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
-
     command = "shell"
 
     @defer.inlineCallbacks

tests/test_command_version.py

@@ -8,7 +8,6 @@ from scrapy.utils.testproc import ProcessTest
 
 
 class VersionTest(ProcessTest, unittest.TestCase):
-
     command = "version"
 
     @defer.inlineCallbacks

tests/test_commands.py

@@ -223,7 +223,6 @@ def get_permissions_dict(
 
 
 class StartprojectTemplatesTest(ProjectTest):
-
     maxDiff = None
 
     def setUp(self):
@@ -604,7 +603,6 @@ class MiscCommandsTest(CommandTest):
 
 
 class RunSpiderCommandTest(CommandTest):
-
     spider_filename = "myspider.py"
 
     debug_log_spider = """
@@ -873,7 +871,6 @@ class MySpider(scrapy.Spider):
 
 @skipIf(platform.system() != "Windows", "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
-
     spider_filename = "myspider.pyw"
 
     def setUp(self):

tests/test_downloader_handlers_http2.py

@@ -25,7 +25,6 @@ from tests.test_downloader_handlers import (
 
 @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2TestCase(Https11TestCase):
-
     scheme = "https"
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 

tests/test_downloadermiddleware.py

@@ -16,7 +16,6 @@ from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 
 
 class ManagerTestCase(TestCase):
-
     settings_dict = None
 
     def setUp(self):

tests/test_downloadermiddleware_decompression.py

@@ -17,7 +17,6 @@ def _test_data(formats):
 
 
 class DecompressionMiddlewareTest(TestCase):
-
     test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]
     uncompressed_body, test_responses = _test_data(test_formats)
 

tests/test_downloadermiddleware_httpcache.py

@@ -14,7 +14,6 @@ from scrapy.utils.test import get_crawler
 
 
 class _BaseTest(unittest.TestCase):
-
     storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
     policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
@@ -146,12 +145,10 @@ class DefaultStorageTest(_BaseTest):
 
 
 class DbmStorageTest(DefaultStorageTest):
-
     storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
 
 
 class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
-
     dbm_module = "tests.mocks.dummydbm"
 
     def _get_settings(self, **new_settings):
@@ -165,7 +162,6 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
 
 class FilesystemStorageTest(DefaultStorageTest):
-
     storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
 
@@ -176,7 +172,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
 
 
 class DummyPolicyTest(_BaseTest):
-
     policy_class = "scrapy.extensions.httpcache.DummyPolicy"
 
     def test_middleware(self):
@@ -270,7 +265,6 @@ class DummyPolicyTest(_BaseTest):
 
 
 class RFC2616PolicyTest(DefaultStorageTest):
-
     policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
     def _process_requestresponse(self, mw, request, response):

tests/test_downloadermiddleware_httpproxy.py

@@ -13,7 +13,6 @@ spider = Spider("foo")
 
 
 class TestHttpProxyMiddleware(TestCase):
-
     failureException = AssertionError
 
     def setUp(self):

tests/test_downloadermiddleware_retry.py

@@ -127,7 +127,6 @@ class RetryTest(unittest.TestCase):
 
 
 class MaxRetryTimesTest(unittest.TestCase):
-
     invalid_url = "http://www.scrapytest.org/invalid_url"
 
     def get_spider_and_middleware(self, settings=None):

tests/test_engine.py

@@ -243,7 +243,6 @@ class CrawlerRun:
 class EngineTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_crawler(self):
-
         for spider in (
             TestSpider,
             DictItemsSpider,

tests/test_exporters.py

@@ -55,7 +55,6 @@ class CustomFieldDataclass:
 
 
 class BaseItemExporterTest(unittest.TestCase):
-
     item_class = TestItem
     custom_field_item_class = CustomFieldItem
 
@@ -513,13 +512,11 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
 
 class XmlItemExporterDataclassTest(XmlItemExporterTest):
-
     item_class = TestDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
-
     _expected_nested = {
         "name": "Jesus",
         "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}},
@@ -559,13 +556,11 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
 
 
 class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
-
     item_class = TestDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
 class JsonItemExporterTest(JsonLinesItemExporterTest):
-
     _expected_nested = [JsonLinesItemExporterTest._expected_nested]
 
     def _get_exporter(self, **kwargs):
@@ -627,13 +622,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
 
 
 class JsonItemExporterDataclassTest(JsonItemExporterTest):
-
     item_class = TestDataClass
     custom_field_item_class = CustomFieldDataclass
 
 
 class CustomExporterItemTest(unittest.TestCase):
-
     item_class = TestItem
 
     def setUp(self):
@@ -664,7 +657,6 @@ class CustomExporterItemTest(unittest.TestCase):
 
 
 class CustomExporterDataclassTest(CustomExporterItemTest):
-
     item_class = TestDataClass
 
 

tests/test_feedexport.py

@@ -1068,7 +1068,6 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
-
         items = [
             self.MyItem({"foo": "bar1", "egg": "spam1"}),
             self.MyItem2({"hello": "world2", "foo": "bar2"}),
@@ -1711,7 +1710,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_gzip_plugin(self):
-
         filename = self._named_tempfile("gzip_file")
 
         settings = {
@@ -1731,7 +1729,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_gzip_plugin_compresslevel(self):
-
         filename_to_compressed = {
             self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
                 self.expected, compresslevel=0
@@ -1839,7 +1836,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin(self):
-
         filename = self._named_tempfile("lzma_file")
 
         settings = {
@@ -1859,7 +1855,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_format(self):
-
         filename_to_compressed = {
             self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
                 self.expected, format=lzma.FORMAT_XZ
@@ -1893,7 +1888,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_check(self):
-
         filename_to_compressed = {
             self._named_tempfile("check_CHECK_NONE"): lzma.compress(
                 self.expected, check=lzma.CHECK_NONE
@@ -1927,7 +1921,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_preset(self):
-
         filename_to_compressed = {
             self._named_tempfile("preset_PRESET_0"): lzma.compress(
                 self.expected, preset=0
@@ -1986,7 +1979,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_bz2_plugin(self):
-
         filename = self._named_tempfile("bz2_file")
 
         settings = {
@@ -2006,7 +1998,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_bz2_plugin_compresslevel(self):
-
         filename_to_compressed = {
             self._named_tempfile("compresslevel_1"): bz2.compress(
                 self.expected, compresslevel=1
@@ -2056,7 +2047,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_custom_plugin_with_parameter(self):
-
         expected = b"foo\r\n\nbar\r\n\n"
         filename = self._named_tempfile("newline")
 
@@ -2075,7 +2065,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_custom_plugin_with_compression(self):
-
         expected = b"foo\r\n\nbar\r\n\n"
 
         filename_to_decompressor = {
@@ -2555,7 +2544,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
         ]
 
         class CustomS3FeedStorage(S3FeedStorage):
-
             stubs = []
 
             def open(self, *args, **kwargs):
@@ -2828,7 +2816,6 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
 
 class URIParamsTest:
-
     spider_name = "uri_params_spider"
     deprecated_options = False
 

tests/test_http_request.py

@@ -19,7 +19,6 @@ from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
-
     request_class = Request
     default_method = "GET"
     default_headers = {}
@@ -424,7 +423,6 @@ class RequestTest(unittest.TestCase):
 
 
 class FormRequestTest(RequestTest):
-
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
@@ -1447,7 +1445,6 @@ def _qs(req, encoding="utf-8", to_unicode=False):
 
 
 class XmlRpcRequestTest(RequestTest):
-
     request_class = XmlRpcRequest
     default_method = "POST"
     default_headers = {b"Content-Type": [b"text/xml"]}

tests/test_http_response.py

@@ -23,7 +23,6 @@ from tests import get_testdata
 
 
 class BaseResponseTest(unittest.TestCase):
-
     response_class = Response
 
     def test_init(self):
@@ -349,7 +348,6 @@ class BaseResponseTest(unittest.TestCase):
 
 
 class TextResponseTest(BaseResponseTest):
-
     response_class = TextResponse
 
     def test_replace(self):
@@ -835,11 +833,9 @@ class TextResponseTest(BaseResponseTest):
 
 
 class HtmlResponseTest(TextResponseTest):
-
     response_class = HtmlResponse
 
     def test_html_encoding(self):
-
         body = b"""<html><head><title>Some page</title>
         <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
         </head><body>Price: \xa3100</body></html>'
@@ -878,7 +874,6 @@ class HtmlResponseTest(TextResponseTest):
 
 
 class XmlResponseTest(TextResponseTest):
-
     response_class = XmlResponse
 
     def test_xml_encoding(self):

tests/test_loader.py

@@ -87,7 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase):
 
 
 class InitializationTestMixin:
-
     item_class = None
 
     def test_keep_single_value(self):

tests/test_loader_deprecated.py

@@ -469,7 +469,6 @@ class BasicItemLoaderTest(unittest.TestCase):
 
 
 class InitializationFromDictTest(unittest.TestCase):
-
     item_class = dict
 
     def test_keep_single_value(self):

tests/test_logformatter.py

@@ -16,7 +16,6 @@ from tests.spiders import ItemSpider
 
 
 class CustomItem(Item):
-
     name = Field()
 
     def __str__(self):

tests/test_pipeline_crawl.py

@@ -116,7 +116,6 @@ class FileDownloadCrawlTestCase(TestCase):
                 self.assertTrue((self.tmpmediastore / i["path"]).exists())
 
     def _assert_files_download_failure(self, crawler, items, code, logs):
-
         # check that the item does NOT have the "images/files" field populated
         self.assertEqual(len(items), 1)
         self.assertIn(self.media_key, items[0])
@@ -205,7 +204,6 @@ else:
 
 
 class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
-
     skip = skip_pillow
 
     pipeline_class = "scrapy.pipelines.images.ImagesPipeline"

tests/test_pipeline_images.py

@@ -33,7 +33,6 @@ else:
 
 
 class ImagesPipelineTestCase(unittest.TestCase):
-
     skip = skip_pillow
 
     def setUp(self):
@@ -325,7 +324,6 @@ class DeprecatedImagesPipeline(ImagesPipeline):
 
 
 class ImagesPipelineTestCaseFieldsMixin:
-
     skip = skip_pillow
 
     def test_item_fields_default(self):
@@ -420,7 +418,6 @@ class ImagesPipelineTestCaseFieldsAttrsItem(
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
-
     skip = skip_pillow
 
     img_cls_attribute_names = [

tests/test_pipeline_media.py

@@ -37,7 +37,6 @@ def _mocked_download_func(request, info):
 
 
 class BaseMediaPipelineTestCase(unittest.TestCase):
-
     pipeline_class = MediaPipeline
     settings = None
 
@@ -213,7 +212,6 @@ class MockedMediaPipeline(MediaPipeline):
 
 
 class MediaPipelineTestCase(BaseMediaPipelineTestCase):
-
     pipeline_class = MockedMediaPipeline
 
     def _callback(self, result):

tests/test_request_cb_kwargs.py

@@ -154,7 +154,6 @@ class KeywordArgumentsSpider(MockServerSpider):
 
 
 class CallbackKeywordArgumentsTestCase(TestCase):
-
     maxDiff = None
 
     def setUp(self):

tests/test_scheduler.py

@@ -43,7 +43,6 @@ class MockDownloader:
 
 class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
-
         settings = dict(
             SCHEDULER_DEBUG=False,
             SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue",
@@ -325,7 +324,6 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):
     @defer.inlineCallbacks
     def test_integration_downloader_aware_priority_queue(self):
         with MockServer() as mockserver:
-
             url = mockserver.url("/status?n=200", is_secure=False)
             start_urls = [url] * 6
             yield self.crawler.crawl(start_urls)

tests/test_settings/__init__.py

@@ -93,7 +93,6 @@ class BaseSettingsTest(unittest.TestCase):
         with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object(
             attr, "set"
         ) as mock_set:
-
             self.settings.attributes = {"TEST_OPTION": attr}
 
             for priority in (0, 10, 20):

tests/test_spider.py

@@ -26,7 +26,6 @@ from tests import get_testdata
 
 
 class SpiderTest(unittest.TestCase):
-
     spider_class = Spider
 
     def setUp(self):
@@ -115,12 +114,10 @@ class SpiderTest(unittest.TestCase):
 
 
 class InitSpiderTest(SpiderTest):
-
     spider_class = InitSpider
 
 
 class XMLFeedSpiderTest(SpiderTest):
-
     spider_class = XMLFeedSpider
 
     def test_register_namespace(self):
@@ -174,7 +171,6 @@ class XMLFeedSpiderTest(SpiderTest):
 
 
 class CSVFeedSpiderTest(SpiderTest):
-
     spider_class = CSVFeedSpider
 
     def test_parse_rows(self):
@@ -196,7 +192,6 @@ class CSVFeedSpiderTest(SpiderTest):
 
 
 class CrawlSpiderTest(SpiderTest):
-
     test_body = b"""<html><head><title>Page title<title>
     <body>
     <p><a href="item/12.html">Item 12</a></p>
@@ -210,7 +205,6 @@ class CrawlSpiderTest(SpiderTest):
     spider_class = CrawlSpider
 
     def test_rule_without_link_extractor(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -234,7 +228,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_links(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -261,7 +254,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_links_filter(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -290,7 +282,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_links_generator(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -318,7 +309,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_request(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -347,7 +337,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_request_with_response(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -383,7 +372,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_request_instance_method(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -410,7 +398,6 @@ class CrawlSpiderTest(SpiderTest):
         )
 
     def test_process_request_instance_method_with_response(self):
-
         response = HtmlResponse(
             "http://example.org/somepage/index.html", body=self.test_body
         )
@@ -467,7 +454,6 @@ class CrawlSpiderTest(SpiderTest):
 
 
 class SitemapSpiderTest(SpiderTest):
-
     spider_class = SitemapSpider
 
     BODY = b"SITEMAP"
@@ -689,7 +675,6 @@ class DeprecationTest(unittest.TestCase):
 
 
 class NoParseMethodSpiderTest(unittest.TestCase):
-
     spider_class = Spider
 
     def test_undefined_parse_method(self):

tests/test_spiderloader/__init__.py

@@ -114,13 +114,11 @@ class SpiderLoaderTest(unittest.TestCase):
         self.assertEqual(crawler.spidercls.name, "spider1")
 
     def test_bad_spider_modules_exception(self):
-
         module = "tests.test_spiderloader.test_spiders.doesnotexist"
         settings = Settings({"SPIDER_MODULES": [module]})
         self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
 
     def test_bad_spider_modules_warning(self):
-
         with warnings.catch_warnings(record=True) as w:
             module = "tests.test_spiderloader.test_spiders.doesnotexist"
             settings = Settings(

tests/test_spidermiddleware_referer.py

@@ -31,7 +31,6 @@ from scrapy.spiders import Spider
 
 
 class TestRefererMiddleware(TestCase):
-
     req_meta = {}
     resp_headers = {}
     settings = {}
@@ -51,7 +50,6 @@ class TestRefererMiddleware(TestCase):
         return Response(origin, headers=self.resp_headers)
 
     def test(self):
-
         for origin, target, referrer in self.scenarii:
             response = self.get_response(origin)
             request = self.get_request(target)
@@ -770,7 +768,6 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
 
 
 class TestRequestMetaSettingFallback(TestCase):
-
     params = [
         (
             # When an unknown policy is referenced in Request.meta
@@ -824,7 +821,6 @@ class TestRequestMetaSettingFallback(TestCase):
     ]
 
     def test(self):
-
         origin = "http://www.scrapy.org"
         target = "http://www.example.com"
 
@@ -923,7 +919,6 @@ class TestPolicyHeaderPrecedence004(
 
 
 class TestReferrerOnRedirect(TestRefererMiddleware):
-
     settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
     scenarii = [
         (
@@ -966,7 +961,6 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
         self.redirectmw = RedirectMiddleware(settings)
 
     def test(self):
-
         for (
             parent,
             target,

tests/test_squeues.py

@@ -73,7 +73,6 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
 
 
 class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
-
     chunksize = 100000
 
     def queue(self):

tests/test_utils_console.py

@@ -20,7 +20,6 @@ except ImportError:
 
 class UtilsConsoleTestCase(unittest.TestCase):
     def test_get_shell_embed_func(self):
-
         shell = get_shell_embed_func(["invalid"])
         self.assertEqual(shell, None)
 
@@ -30,14 +29,12 @@ class UtilsConsoleTestCase(unittest.TestCase):
 
     @unittest.skipIf(not bpy, "bpython not available in testenv")
     def test_get_shell_embed_func2(self):
-
         shell = get_shell_embed_func(["bpython"])
         self.assertTrue(callable(shell))
         self.assertEqual(shell.__name__, "_embed_bpython_shell")
 
     @unittest.skipIf(not ipy, "IPython not available in testenv")
     def test_get_shell_embed_func3(self):
-
         # default shell should be 'ipython'
         shell = get_shell_embed_func()
         self.assertEqual(shell.__name__, "_embed_ipython_shell")

tests/test_utils_iterators.py

@@ -7,7 +7,6 @@ from tests import get_testdata
 
 
 class XmliterTestCase(unittest.TestCase):
-
     xmliter = staticmethod(xmliter)
 
     def test_xmliter(self):

tests/test_utils_request.py

@@ -462,7 +462,6 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
         warning to be logged."""
 
         class RequestFingerprinter:
-
             cache = WeakKeyDictionary()
 
             def fingerprint(self, request):
@@ -641,7 +640,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
 
     def test_from_crawler_and_settings(self):
         class RequestFingerprinter:
-
             # This method is ignored due to the presence of from_crawler
             @classmethod
             def from_settings(cls, settings):

tests/test_utils_template.py

@@ -16,7 +16,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
         rmtree(self.tmp_path)
 
     def test_simple_render(self):
-
         context = dict(project_name="proj", name="spi", classname="TheSpider")
         template = "from ${project_name}.spiders.${name} import ${classname}"
         rendered = "from proj.spiders.spi import TheSpider"

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = security,flake8,black,typing,py
+envlist = pre-commit,typing,py
 minversion = 1.7.0
 
 [testenv]
@@ -37,44 +37,34 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    lxml-stubs==0.2.0
+    lxml-stubs==0.4.0
     mypy==0.991
     types-attrs==19.1.0
-    types-pyOpenSSL==21.0.0
+    types-pyOpenSSL==23.0.0.2
-    types-setuptools==57.0.0
+    types-setuptools==65.7.0.3
 commands =
     mypy --show-error-codes {posargs: scrapy tests}
 
-[testenv:security]
+[testenv:pre-commit]
 basepython = python3
 deps =
-    bandit==1.7.4
+    pre-commit
 commands =
-    bandit -r -c .bandit.yml {posargs:scrapy}
+    pre-commit run {posargs:--all-files}
-
-[testenv:flake8]
-basepython = python3
-deps =
-    {[testenv]deps}
-    # Twisted[http2] is required to import some files
-    Twisted[http2]>=17.9.0
-    flake8==6.0.0
-commands =
-    flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.15.6
+    pylint==2.16.0
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==4.0.1
+    twine==4.0.2
-    build==0.9.0
+    build==0.10.0
 commands =
     python -m build --sdist
     twine check dist/*
@@ -195,9 +185,3 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:black]
-deps =
-    black==22.12.0
-commands =
-    black {posargs:--check .}