Bump black.

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 
 max-line-length = 119
-ignore = W503, E203
+ignore = E203, E501, E701, E704, W503
 
 exclude =
     docs/conf.py

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
-  rev: 23.9.1
+  rev: 24.2.0
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
@@ -21,4 +21,4 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==23.9.1
+    - black==24.2.0

docs/topics/addons.rst

@@ -150,8 +150,7 @@ Access the crawler instance:
         def from_crawler(cls, crawler):
             return cls(crawler)
 
-        def update_settings(self, settings):
+        def update_settings(self, settings): ...
-            ...
 
 Use a fallback component:
 

scrapy/commands/__init__.py

@@ -1,6 +1,7 @@
 """
 Base class for Scrapy commands
 """
+
 import argparse
 import os
 from pathlib import Path

scrapy/commands/shell.py

@@ -3,6 +3,7 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
+
 from argparse import Namespace
 from threading import Thread
 from typing import List, Type

scrapy/core/downloader/handlers/__init__.py

@@ -21,9 +21,9 @@ logger = logging.getLogger(__name__)
 class DownloadHandlers:
     def __init__(self, crawler: "Crawler"):
         self._crawler: "Crawler" = crawler
-        self._schemes: Dict[
+        self._schemes: Dict[str, Union[str, Callable]] = (
-            str, Union[str, Callable]
+            {}
-        ] = {}  # stores acceptable schemes on instancing
+        )  # stores acceptable schemes on instancing
         self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes
         self._notconfigured: Dict[str, str] = {}  # remembers failed handlers
         handlers: Dict[str, Union[str, Callable]] = without_none_values(

scrapy/core/downloader/handlers/http10.py

@@ -1,5 +1,6 @@
 """Download handlers for http and https schemes
 """
+
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
 

scrapy/core/downloader/middleware.py

@@ -3,6 +3,7 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from typing import Any, Callable, Generator, List, Union, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks

scrapy/core/engine.py

@@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
 For more information see docs/topics/architecture.rst
 
 """
+
 import logging
 from time import time
 from typing import (

scrapy/core/http2/stream.py

@@ -111,17 +111,17 @@ class Stream:
         # Metadata of an HTTP/2 connection stream
         # initialized when stream is instantiated
         self.metadata: Dict = {
-            "request_content_length": 0
+            "request_content_length": (
-            if self._request.body is None
+                0 if self._request.body is None else len(self._request.body)
-            else len(self._request.body),
+            ),
             # Flag to keep track whether the stream has initiated the request
             "request_sent": False,
             # Flag to track whether we have logged about exceeding download warnsize
             "reached_warnsize": False,
             # Each time we send a data frame, we will decrease value by the amount send.
-            "remaining_content_length": 0
+            "remaining_content_length": (
-            if self._request.body is None
+                0 if self._request.body is None else len(self._request.body)
-            else len(self._request.body),
+            ),
             # Flag to keep track whether client (self) have closed this stream
             "stream_closed_local": False,
             # Flag to keep track whether the server has closed the stream

scrapy/core/scraper.py

@@ -1,5 +1,6 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
+
 from __future__ import annotations
 
 import logging

scrapy/core/spidermw.py

@@ -3,6 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice

scrapy/downloadermiddlewares/defaultheaders.py

@@ -3,6 +3,7 @@ DefaultHeaders downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterable, Tuple, Union

scrapy/downloadermiddlewares/downloadtimeout.py

@@ -3,6 +3,7 @@ Download timeout middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Union

scrapy/downloadermiddlewares/retry.py

@@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+
 from __future__ import annotations
 
 import warnings

scrapy/exceptions.py

@@ -4,6 +4,7 @@ Scrapy core exceptions
 These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
+
 from typing import Any
 
 # Internal

scrapy/extension.py

@@ -3,6 +3,7 @@ The Extension Manager
 
 See documentation in docs/topics/extensions.rst
 """
+
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 

scrapy/extensions/corestats.py

@@ -1,6 +1,7 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
+
 from datetime import datetime, timezone
 
 from scrapy import signals

scrapy/extensions/memusage.py

@@ -3,6 +3,7 @@ MemoryUsage extension
 
 See documentation in docs/topics/extensions.rst
 """
+
 import logging
 import socket
 import sys

scrapy/extensions/postprocessing.py

@@ -1,6 +1,7 @@
 """
 Extension for processing data before they are exported to feeds.
 """
+
 from bz2 import BZ2File
 from gzip import GzipFile
 from io import IOBase

scrapy/http/request/__init__.py

@@ -4,6 +4,7 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+
 import inspect
 from typing import (
     Any,
@@ -231,12 +232,16 @@ class Request(object_ref):
         """
         d = {
             "url": self.url,  # urls are safe (safe_string_url)
-            "callback": _find_method(spider, self.callback)
+            "callback": (
-            if callable(self.callback)
+                _find_method(spider, self.callback)
-            else self.callback,
+                if callable(self.callback)
-            "errback": _find_method(spider, self.errback)
+                else self.callback
-            if callable(self.errback)
+            ),
-            else self.errback,
+            "errback": (
+                _find_method(spider, self.errback)
+                if callable(self.errback)
+                else self.errback
+            ),
             "headers": dict(self.headers),
         }
         for attr in self.attributes:

scrapy/http/request/rpc.py

@@ -4,6 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
+
 import xmlrpc.client as xmlrpclib
 from typing import Any, Optional
 

scrapy/http/response/__init__.py

@@ -4,6 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+
 from __future__ import annotations
 
 from ipaddress import IPv4Address, IPv6Address

scrapy/http/response/text.py

@@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class.
 
 See documentation in docs/topics/request-response.rst
 """
+
 from __future__ import annotations
 
 import json

scrapy/link.py

@@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+
 from typing import Any
 
 

scrapy/linkextractors/__init__.py

@@ -5,6 +5,7 @@ This package contains a collection of Link Extractors.
 
 For more info see docs/topics/link-extractors.rst
 """
+
 import re
 
 # common file extensions that are not followed if they occur in links

scrapy/linkextractors/lxmlhtml.py

@@ -1,6 +1,7 @@
 """
 Link extractor based on lxml.html
 """
+
 import logging
 import operator
 from functools import partial

scrapy/loader/__init__.py

@@ -3,6 +3,7 @@ Item Loader
 
 See documentation in docs/topics/loaders.rst
 """
+
 import itemloaders
 
 from scrapy.item import Item

scrapy/mail.py

@@ -3,6 +3,7 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+
 import logging
 from email import encoders as Encoders
 from email.mime.base import MIMEBase

scrapy/pipelines/__init__.py

@@ -3,6 +3,7 @@ Item pipeline
 
 See documentation in docs/item-pipeline.rst
 """
+
 from typing import Any, List
 
 from twisted.internet.defer import Deferred

scrapy/pipelines/files.py

@@ -3,6 +3,7 @@ Files Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
+
 import base64
 import functools
 import hashlib

scrapy/pipelines/images.py

@@ -3,6 +3,7 @@ Images Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
+
 import functools
 import hashlib
 import warnings

scrapy/responsetypes.py

@@ -2,6 +2,7 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
+
 from io import StringIO
 from mimetypes import MimeTypes
 from pkgutil import get_data

scrapy/selector/unified.py

@@ -1,6 +1,7 @@
 """
 XPath selectors based on lxml
 """
+
 from typing import Any, Optional, Type, Union
 
 from parsel import Selector as _ParselSelector

scrapy/settings/__init__.py

@@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int:
 
 
 class SettingsAttribute:
-
     """Class for storing data related to settings attributes.
 
     This class is intended for internal usage, you should try Settings class

scrapy/shell.py

@@ -3,6 +3,7 @@
 See documentation in docs/topics/shell.rst
 
 """
+
 import os
 import signal
 

scrapy/spidermiddlewares/httperror.py

@@ -3,6 +3,7 @@ HttpError Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/offsite.py

@@ -3,6 +3,7 @@ Offsite Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/referer.py

@@ -2,6 +2,7 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
+
 from __future__ import annotations
 
 import warnings

scrapy/spiders/__init__.py

@@ -3,6 +3,7 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spiders/feed.py

@@ -4,6 +4,7 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
+
 from scrapy.exceptions import NotConfigured, NotSupported
 from scrapy.selector import Selector
 from scrapy.spiders import Spider

scrapy/statscollectors.py

@@ -1,6 +1,7 @@
 """
 Scrapy extension for collecting scraping stats
 """
+
 import logging
 import pprint
 from typing import TYPE_CHECKING, Any, Dict, Optional

scrapy/utils/defer.py

@@ -1,6 +1,7 @@
 """
 Helper functions for dealing with Twisted deferreds
 """
+
 import asyncio
 import inspect
 from asyncio import Future
@@ -304,13 +305,11 @@ _T = TypeVar("_T")
 
 
 @overload
-def deferred_from_coro(o: _CT) -> Deferred:
+def deferred_from_coro(o: _CT) -> Deferred: ...
-    ...
 
 
 @overload
-def deferred_from_coro(o: _T) -> _T:
+def deferred_from_coro(o: _T) -> _T: ...
-    ...
 
 
 def deferred_from_coro(o: _T) -> Union[Deferred, _T]:

scrapy/utils/deprecate.py

@@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = []
 
 
 @overload
-def update_classpath(path: str) -> str:
+def update_classpath(path: str) -> str: ...
-    ...
 
 
 @overload
-def update_classpath(path: Any) -> Any:
+def update_classpath(path: Any) -> Any: ...
-    ...
 
 
 def update_classpath(path: Any) -> Any:

scrapy/utils/iterators.py

@@ -225,18 +225,17 @@ def csviter(
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes]) -> str:
+def _body_or_str(obj: Union[Response, str, bytes]) -> str: ...
-    ...
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str:
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ...
-    ...
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes:
+def _body_or_str(
-    ...
+    obj: Union[Response, str, bytes], unicode: Literal[False]
+) -> bytes: ...
 
 
 def _body_or_str(

scrapy/utils/misc.py

@@ -1,4 +1,5 @@
 """Helper functions which don't fit anywhere else"""
+
 import ast
 import hashlib
 import inspect

scrapy/utils/python.py

@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+
 import collections.abc
 import gc
 import inspect
@@ -285,13 +286,11 @@ def equal_attributes(
 
 
 @overload
-def without_none_values(iterable: Mapping) -> dict:
+def without_none_values(iterable: Mapping) -> dict: ...
-    ...
 
 
 @overload
-def without_none_values(iterable: Iterable) -> Iterable:
+def without_none_values(iterable: Iterable) -> Iterable: ...
-    ...
 
 
 def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:

scrapy/utils/request.py

@@ -44,7 +44,9 @@ def _serialize_headers(
             yield from request.headers.getlist(header)
 
 
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache: (
+    "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+)
 _fingerprint_cache = WeakKeyDictionary()
 
 
@@ -114,8 +116,7 @@ def fingerprint(
 
 
 class RequestFingerprinterProtocol(Protocol):
-    def fingerprint(self, request: Request) -> bytes:
+    def fingerprint(self, request: Request) -> bytes: ...
-        ...
 
 
 class RequestFingerprinter:

scrapy/utils/response.py

@@ -2,6 +2,7 @@
 This module provides some useful functions for working with
 scrapy.http.Response objects
 """
+
 import os
 import re
 import tempfile
@@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     return _baseurl_cache[response]
 
 
-_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = (
+_metaref_cache: (
-    WeakKeyDictionary()
+    "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]"
-)
+) = WeakKeyDictionary()
 
 
 def get_meta_refresh(

scrapy/utils/signal.py

@@ -1,4 +1,5 @@
 """Helper functions for working with signals"""
+
 import collections.abc
 import logging
 from typing import Any as TypingAny

scrapy/utils/sitemap.py

@@ -4,6 +4,7 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+
 from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 

scrapy/utils/spider.py

@@ -39,13 +39,11 @@ def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator:  # type: ig
 
 
 @overload
-def iterate_spider_output(result: CoroutineType) -> Deferred:
+def iterate_spider_output(result: CoroutineType) -> Deferred: ...
-    ...
 
 
 @overload
-def iterate_spider_output(result: _T) -> Iterable:
+def iterate_spider_output(result: _T) -> Iterable: ...
-    ...
 
 
 def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
@@ -83,8 +81,7 @@ def spidercls_for_request(
     default_spidercls: Type[Spider],
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Type[Spider]:
+) -> Type[Spider]: ...
-    ...
 
 
 @overload
@@ -94,8 +91,7 @@ def spidercls_for_request(
     default_spidercls: Literal[None],
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Optional[Type[Spider]]:
+) -> Optional[Type[Spider]]: ...
-    ...
 
 
 @overload
@@ -105,8 +101,7 @@ def spidercls_for_request(
     *,
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Optional[Type[Spider]]:
+) -> Optional[Type[Spider]]: ...
-    ...
 
 
 def spidercls_for_request(

scrapy/utils/url.py

@@ -5,6 +5,7 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
+
 import re
 from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse

tests/mocks/dummydbm.py

@@ -1,4 +1,5 @@
 """DBM-like dummy module"""
+
 import collections
 from typing import Any, DefaultDict
 

tests/spiders.py

@@ -1,6 +1,7 @@
 """
 Some spiders used for testing and benchmarking
 """
+
 import asyncio
 import time
 from urllib.parse import urlencode

tests/test_exporters.py

@@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, "John\xa3")
 
         ie = self._get_exporter(fields_to_export={"name": "名稱"})
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")])
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")]
+        )
 
     def test_field_custom_serializer(self):
         i = self.custom_field_item_class(name="John\xa3", age="22")

tests/test_pipeline_media.py

@@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler
 try:
     from PIL import Image  # noqa: imported just to check for the import error
 except ImportError:
-    skip_pillow: Optional[
+    skip_pillow: Optional[str] = (
-        str
+        "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
-    ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+    )
 else:
     skip_pillow = None
 

tests/test_responsetypes.py

@@ -33,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase):
             ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
             ("attachment;filename=data高.doc".encode("gbk"), Response),
             ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
-            ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse),
+            (
+                "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),
+                XmlResponse,
+            ),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)

tests/test_webclient.py

@@ -2,6 +2,7 @@
 from twisted.internet import defer
 Tests borrowed from the twisted.web.client tests.
 """
+
 import shutil
 from pathlib import Path
 from tempfile import mkdtemp