From 282767f23b2e71969bea3bd5492abde88d2054c6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:49:06 +0500 Subject: [PATCH] Bump black. --- .flake8 | 2 +- .pre-commit-config.yaml | 4 ++-- docs/topics/addons.rst | 3 +-- scrapy/commands/__init__.py | 1 + scrapy/commands/shell.py | 1 + scrapy/core/downloader/handlers/__init__.py | 6 +++--- scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/middleware.py | 1 + scrapy/core/engine.py | 1 + scrapy/core/http2/stream.py | 12 ++++++------ scrapy/core/scraper.py | 1 + scrapy/core/spidermw.py | 1 + scrapy/downloadermiddlewares/defaultheaders.py | 1 + scrapy/downloadermiddlewares/downloadtimeout.py | 1 + scrapy/downloadermiddlewares/retry.py | 1 + scrapy/exceptions.py | 1 + scrapy/extension.py | 1 + scrapy/extensions/corestats.py | 1 + scrapy/extensions/memusage.py | 1 + scrapy/extensions/postprocessing.py | 1 + scrapy/http/request/__init__.py | 17 +++++++++++------ scrapy/http/request/rpc.py | 1 + scrapy/http/response/__init__.py | 1 + scrapy/http/response/text.py | 1 + scrapy/link.py | 1 + scrapy/linkextractors/__init__.py | 1 + scrapy/linkextractors/lxmlhtml.py | 1 + scrapy/loader/__init__.py | 1 + scrapy/mail.py | 1 + scrapy/pipelines/__init__.py | 1 + scrapy/pipelines/files.py | 1 + scrapy/pipelines/images.py | 1 + scrapy/responsetypes.py | 1 + scrapy/selector/unified.py | 1 + scrapy/settings/__init__.py | 1 - scrapy/shell.py | 1 + scrapy/spidermiddlewares/httperror.py | 1 + scrapy/spidermiddlewares/offsite.py | 1 + scrapy/spidermiddlewares/referer.py | 1 + scrapy/spiders/__init__.py | 1 + scrapy/spiders/feed.py | 1 + scrapy/statscollectors.py | 1 + scrapy/utils/defer.py | 7 +++---- scrapy/utils/deprecate.py | 6 ++---- scrapy/utils/iterators.py | 11 +++++------ scrapy/utils/misc.py | 1 + scrapy/utils/python.py | 7 +++---- scrapy/utils/request.py | 7 ++++--- scrapy/utils/response.py | 7 ++++--- scrapy/utils/signal.py | 1 + scrapy/utils/sitemap.py | 1 + scrapy/utils/spider.py | 15 +++++---------- scrapy/utils/url.py | 1 + tests/mocks/dummydbm.py | 1 + tests/spiders.py | 1 + tests/test_exporters.py | 4 +++- tests/test_pipeline_media.py | 6 +++--- tests/test_responsetypes.py | 5 ++++- tests/test_webclient.py | 1 + 59 files changed, 102 insertions(+), 60 deletions(-) diff --git a/.flake8 b/.flake8 index 544d72956..62ccad9cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 119 -ignore = W503, E203 +ignore = E203, E501, E701, E704, W503 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..83bc65b67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==24.2.0 diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..d2fc41003 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -150,8 +150,7 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2aa569cdd..27993710e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,6 +1,7 @@ """ Base class for Scrapy commands """ + import argparse import os from pathlib import Path diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 12e37babc..f72a23c6a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,6 +3,7 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ + from argparse import Namespace from threading import Thread from typing import List, Type diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 416669b7f..ade51ca63 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -21,9 +21,9 @@ logger = logging.getLogger(__name__) class DownloadHandlers: def __init__(self, crawler: "Crawler"): self._crawler: "Crawler" = crawler - self._schemes: Dict[ - str, Union[str, Callable] - ] = {} # stores acceptable schemes on instancing + self._schemes: Dict[str, Union[str, Callable]] = ( + {} + ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index b6ac7a251..d168c2b2e 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,5 +1,6 @@ """Download handlers for http and https schemes """ + from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dca13c01e..52ebe4e22 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,6 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 545cd401f..2db085081 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ + import logging from time import time from typing import ( diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 0f282d83d..4132fc385 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -111,17 +111,17 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated self.metadata: Dict = { - "request_content_length": 0 - if self._request.body is None - else len(self._request.body), + "request_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether the stream has initiated the request "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - "remaining_content_length": 0 - if self._request.body is None - else len(self._request.body), + "remaining_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether client (self) have closed this stream "stream_closed_local": False, # Flag to keep track whether the server has closed the stream diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8fb16b8a9..272841e01 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,5 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" + from __future__ import annotations import logging diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 031a0be36..1ccfd08a2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 8aec37cf1..58fd415b9 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,6 +3,7 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 1c904c05b..fd7c03a38 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,6 +3,7 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Union diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 3c494de78..46587a898 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ + from __future__ import annotations import warnings diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6d188c489..e7ecdbe0c 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ + from typing import Any # Internal diff --git a/scrapy/extension.py b/scrapy/extension.py index 4e365cfa1..6be14450c 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -3,6 +3,7 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ + from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 302a615f2..717c249d9 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,6 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ + from datetime import datetime, timezone from scrapy import signals diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ca766c938..4d4501c44 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,6 +3,7 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ + import logging import socket import sys diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 17969c5b0..f8b59827b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,6 +1,7 @@ """ Extension for processing data before they are exported to feeds. """ + from bz2 import BZ2File from gzip import GzipFile from io import IOBase diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..6269ee86a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,6 +4,7 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ + import inspect from typing import ( Any, @@ -231,12 +232,16 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) - if callable(self.callback) - else self.callback, - "errback": _find_method(spider, self.errback) - if callable(self.errback) - else self.errback, + "callback": ( + _find_method(spider, self.callback) + if callable(self.callback) + else self.callback + ), + "errback": ( + _find_method(spider, self.errback) + if callable(self.errback) + else self.errback + ), "headers": dict(self.headers), } for attr in self.attributes: diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 84b433990..e20e7c438 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,6 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ + import xmlrpc.client as xmlrpclib from typing import Any, Optional diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..d73dfce4b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations from ipaddress import IPv4Address, IPv6Address diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6596d8a5c..2816610fb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations import json diff --git a/scrapy/link.py b/scrapy/link.py index 0868ae5ef..4bdbc1823 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + from typing import Any diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3774430a7..73a63651c 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -5,6 +5,7 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ + import re # common file extensions that are not followed if they occur in links diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..d76db20ba 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,7 @@ """ Link extractor based on lxml.html """ + import logging import operator from functools import partial diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 1042a3d48..529fa279e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,6 +3,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ + import itemloaders from scrapy.item import Item diff --git a/scrapy/mail.py b/scrapy/mail.py index 237327451..4b18b6003 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,6 +3,7 @@ Mail sending helpers See documentation in docs/topics/email.rst """ + import logging from email import encoders as Encoders from email.mime.base import MIMEBase diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c97d71fb6..f9544d329 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,6 +3,7 @@ Item pipeline See documentation in docs/item-pipeline.rst """ + from typing import Any, List from twisted.internet.defer import Deferred diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 73064ad10..1d7625299 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,6 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ + import base64 import functools import hashlib diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 02c4b1361..8169583f8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,6 +3,7 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ + import functools import hashlib import warnings diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 0d127d851..702e50536 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,6 +2,7 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 863fb6032..75d5e9fbd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,6 +1,7 @@ """ XPath selectors based on lxml """ + from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b5d8fdb12..d270a72f4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int: class SettingsAttribute: - """Class for storing data related to settings attributes. This class is intended for internal usage, you should try Settings class diff --git a/scrapy/shell.py b/scrapy/shell.py index bb3b1461c..05909977a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,6 +3,7 @@ See documentation in docs/topics/shell.rst """ + import os import signal diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94450b35b..35c869a75 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,6 +3,7 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a5214702d..dd2fccfcb 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,6 +3,7 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a29e0ebb5..a0b6851e5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ + from __future__ import annotations import warnings diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e16d71727..72c2aaba7 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 599af7360..5caf8c79e 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,6 +4,7 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ + from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 15193aac5..ab571a3ab 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,6 +1,7 @@ """ Scrapy extension for collecting scraping stats """ + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bf3c5ef5b..c391db9fd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,6 +1,7 @@ """ Helper functions for dealing with Twisted deferreds """ + import asyncio import inspect from asyncio import Future @@ -304,13 +305,11 @@ _T = TypeVar("_T") @overload -def deferred_from_coro(o: _CT) -> Deferred: - ... +def deferred_from_coro(o: _CT) -> Deferred: ... @overload -def deferred_from_coro(o: _T) -> _T: - ... +def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Union[Deferred, _T]: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ea577c44a..e0f2ac763 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = [] @overload -def update_classpath(path: str) -> str: - ... +def update_classpath(path: str) -> str: ... @overload -def update_classpath(path: Any) -> Any: - ... +def update_classpath(path: Any) -> Any: ... def update_classpath(path: Any) -> Any: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c56be5ea2..93a2ba7a1 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -225,18 +225,17 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: - ... +def _body_or_str( + obj: Union[Response, str, bytes], unicode: Literal[False] +) -> bytes: ... def _body_or_str( diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b38190cb3..7b43760a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,5 @@ """Helper functions which don't fit anywhere else""" + import ast import hashlib import inspect diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 68ca96b69..7b408c49c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ + import collections.abc import gc import inspect @@ -285,13 +286,11 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: - ... +def without_none_values(iterable: Mapping) -> dict: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: - ... +def without_none_values(iterable: Iterable) -> Iterable: ... def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index db0b44cf4..e99d1eeb5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,7 +44,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache: ( + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +) _fingerprint_cache = WeakKeyDictionary() @@ -114,8 +116,7 @@ def fingerprint( class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... + def fingerprint(self, request: Request) -> bytes: ... class RequestFingerprinter: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 33cd692bf..63a484b42 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,6 +2,7 @@ This module provides some useful functions for working with scrapy.http.Response objects """ + import os import re import tempfile @@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( - WeakKeyDictionary() -) +_metaref_cache: ( + "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" +) = WeakKeyDictionary() def get_meta_refresh( diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 21a12a19e..a25100c03 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,4 +1,5 @@ """Helper functions for working with signals""" + import collections.abc import logging from typing import Any as TypingAny diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 3d2ecc9a7..8bf941eb2 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,6 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 704df8657..855bc8f87 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -39,13 +39,11 @@ def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ig @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... +def iterate_spider_output(result: CoroutineType) -> Deferred: ... @overload -def iterate_spider_output(result: _T) -> Iterable: - ... +def iterate_spider_output(result: _T) -> Iterable: ... def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: @@ -83,8 +81,7 @@ def spidercls_for_request( default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: - ... +) -> Type[Spider]: ... @overload @@ -94,8 +91,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... @overload @@ -105,8 +101,7 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... def spidercls_for_request( diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 22b4197f9..9d97cb12f 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,6 +5,7 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ + import re from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 2869ff8f7..bde3de228 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,4 +1,5 @@ """DBM-like dummy module""" + import collections from typing import Any, DefaultDict diff --git a/tests/spiders.py b/tests/spiders.py index 3df153a12..94969db99 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,6 +1,7 @@ """ Some spiders used for testing and benchmarking """ + import asyncio import time from urllib.parse import urlencode diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c11913365..59b724495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, "John\xa3") ie = self._get_exporter(fields_to_export={"name": "名稱"}) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")]) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")] + ) def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 820484565..d477b59be 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[ - str - ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + skip_pillow: Optional[str] = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: skip_pillow = None diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 713a83d52..2633cca5b 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -33,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase): ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), - ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse), + ( + "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), + XmlResponse, + ), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 53558814d..cce119001 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,6 +2,7 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ + import shutil from pathlib import Path from tempfile import mkdtemp