sort imports with isort

conftest.py

@@ -4,7 +4,6 @@ import pytest
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
-
 from tests.keys import generate_keys
 
 

docs/_ext/scrapydocs.py

@@ -1,7 +1,8 @@
 from operator import itemgetter
-from docutils.parsers.rst.roles import set_classes
+
 from docutils import nodes
 from docutils.parsers.rst import Directive
+from docutils.parsers.rst.roles import set_classes
 from sphinx.util.nodes import make_refnode
 
 

extras/qps-bench-server.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
-from time import time
 from collections import deque
-from twisted.web.server import Site, NOT_DONE_YET
-from twisted.web.resource import Resource
+from time import time
+
 from twisted.internet import reactor
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET, Site
 
 
 class Root(Resource):

extras/qpsclient.py

@@ -8,8 +8,8 @@ usage:
 
 """
 
-from scrapy.spiders import Spider
 from scrapy.http import Request
+from scrapy.spiders import Spider
 
 
 class QPSSpider(Spider):

scrapy/__init__.py

@@ -8,12 +8,12 @@ import warnings
 
 from twisted import version as _txv
 
+from scrapy.http import FormRequest, Request
+from scrapy.item import Field, Item
+from scrapy.selector import Selector
+
 # Declare top-level shortcuts
 from scrapy.spiders import Spider
-from scrapy.http import Request, FormRequest
-from scrapy.selector import Selector
-from scrapy.item import Item, Field
-
 
 __all__ = [
     "__version__",

scrapy/cmdline.py

@@ -1,16 +1,17 @@
-import sys
-import os
 import argparse
 import cProfile
 import inspect
+import os
+import sys
+
 import pkg_resources
 
 import scrapy
+from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.crawler import CrawlerProcess
-from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
-from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
 
 

scrapy/commands/__init__.py

@@ -1,16 +1,16 @@
 """
 Base class for Scrapy commands
 """
-import os
 import argparse
+import os
 from pathlib import Path
 from typing import Any, Dict, Optional
 
 from twisted.python import failure
-from scrapy.crawler import CrawlerProcess
 
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.crawler import CrawlerProcess
 from scrapy.exceptions import UsageError
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 class ScrapyCommand:

scrapy/commands/bench.py

@@ -1,6 +1,6 @@
+import subprocess
 import sys
 import time
-import subprocess
 from urllib.parse import urlencode
 
 import scrapy

scrapy/commands/check.py

@@ -1,11 +1,12 @@
 import time
 from collections import defaultdict
-from unittest import TextTestRunner, TextTestResult as _TextTestResult
+from unittest import TextTestResult as _TextTestResult
+from unittest import TextTestRunner
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.misc import load_object, set_environ
 
 
 class TextTestResult(_TextTestResult):

scrapy/commands/edit.py

@@ -1,5 +1,5 @@
-import sys
 import os
+import sys
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError

scrapy/commands/fetch.py

@@ -1,11 +1,12 @@
 import sys
+
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
-from scrapy.http import Request
 from scrapy.exceptions import UsageError
+from scrapy.http import Request
 from scrapy.utils.datatypes import SequenceExclude
-from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
 
 class Command(ScrapyCommand):

scrapy/commands/genspider.py

@@ -1,16 +1,15 @@
 import os
 import shutil
 import string
-
-from pathlib import Path
 from importlib import import_module
+from pathlib import Path
 from typing import Optional, cast
 from urllib.parse import urlparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
+from scrapy.utils.template import render_templatefile, string_camelcase
 
 
 def sanitize_module_name(module_name):

scrapy/commands/parse.py

@@ -2,15 +2,15 @@ import json
 import logging
 from typing import Dict
 
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
+from twisted.internet.defer import maybeDeferred
 from w3lib.url import is_url
 
-from twisted.internet.defer import maybeDeferred
 from scrapy.commands import BaseRunSpiderCommand
+from scrapy.exceptions import UsageError
 from scrapy.http import Request
 from scrapy.utils import display
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
-from scrapy.exceptions import UsageError
 
 logger = logging.getLogger(__name__)
 

scrapy/commands/runspider.py

@@ -1,13 +1,13 @@
 import sys
+from importlib import import_module
 from os import PathLike
 from pathlib import Path
-from importlib import import_module
 from types import ModuleType
 from typing import Union
 
-from scrapy.utils.spider import iter_spider_classes
-from scrapy.exceptions import UsageError
 from scrapy.commands import BaseRunSpiderCommand
+from scrapy.exceptions import UsageError
+from scrapy.utils.spider import iter_spider_classes
 
 
 def _import_file(filepath: Union[str, PathLike]) -> ModuleType:

scrapy/commands/shell.py

@@ -8,7 +8,7 @@ from threading import Thread
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
-from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 from scrapy.utils.url import guess_scheme
 
 

scrapy/commands/startproject.py

@@ -1,16 +1,15 @@
-import re
 import os
+import re
 import string
 from importlib.util import find_spec
 from pathlib import Path
-from shutil import ignore_patterns, move, copy2, copystat
+from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
-
+from scrapy.utils.template import render_templatefile, string_camelcase
 
 TEMPLATES_TO_RENDER = (
     ("scrapy.cfg",),

scrapy/commands/view.py

@@ -1,4 +1,5 @@
 import argparse
+
 from scrapy.commands import fetch
 from scrapy.utils.response import open_in_browser
 

scrapy/contracts/default.py

@@ -1,6 +1,6 @@
 import json
 
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
 
 from scrapy.contracts import Contract
 from scrapy.exceptions import ContractFail

scrapy/core/downloader/__init__.py

@@ -1,16 +1,16 @@
 import random
-from time import time
-from datetime import datetime
 from collections import deque
+from datetime import datetime
+from time import time
 
 from twisted.internet import defer, task
 
+from scrapy import signals
+from scrapy.core.downloader.handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.resolver import dnscache
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.resolver import dnscache
-from scrapy import signals
-from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
-from scrapy.core.downloader.handlers import DownloadHandlers
 
 
 class Slot:

scrapy/core/downloader/contextfactory.py

@@ -3,10 +3,10 @@ import warnings
 from OpenSSL import SSL
 from twisted.internet._sslverify import _setAcceptableProtocols
 from twisted.internet.ssl import (
-    optionsForClientTLS,
-    CertificateOptions,
-    platformTrust,
     AcceptableCiphers,
+    CertificateOptions,
+    optionsForClientTLS,
+    platformTrust,
 )
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
@@ -15,8 +15,8 @@ from zope.interface.verify import verifyObject
 
 from scrapy.core.downloader.tls import (
     DEFAULT_CIPHERS,
-    openssl_methods,
     ScrapyClientTLSOptions,
+    openssl_methods,
 )
 from scrapy.utils.misc import create_instance, load_object
 

scrapy/core/downloader/handlers/__init__.py

@@ -10,7 +10,6 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/core/downloader/handlers/http11.py

@@ -13,15 +13,15 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.python.failure import Failure
 from twisted.web.client import (
+    URI,
     Agent,
     HTTPConnectionPool,
     ResponseDone,
     ResponseFailed,
-    URI,
 )
-from twisted.web.http import _DataLoss, PotentialDataLoss
+from twisted.web.http import PotentialDataLoss, _DataLoss
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
 from zope.interface import implementer
 
 from scrapy import signals

scrapy/core/downloader/handlers/http2.py

@@ -16,7 +16,6 @@ from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 
-
 H2DownloadHandlerOrSubclass = TypeVar(
     "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
 )

scrapy/core/downloader/middleware.py

@@ -12,8 +12,8 @@ from scrapy import Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
 
 
 class DownloaderMiddlewareManager(MiddlewareManager):

scrapy/core/downloader/tls.py

@@ -4,12 +4,12 @@ from OpenSSL import SSL
 from service_identity.exceptions import CertificateError
 from twisted.internet._sslverify import (
     ClientTLSOptions,
-    verifyHostname,
     VerificationError,
+    verifyHostname,
 )
 from twisted.internet.ssl import AcceptableCiphers
 
-from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
+from scrapy.utils.ssl import get_temp_key_info, x509name_to_string
 
 logger = logging.getLogger(__name__)
 

scrapy/core/downloader/webclient.py

@@ -1,15 +1,15 @@
 import re
 from time import time
-from urllib.parse import urlparse, urlunparse, urldefrag
-from twisted.web.http import HTTPClient
+from urllib.parse import urldefrag, urlparse, urlunparse
 
 from twisted.internet import defer
 from twisted.internet.protocol import ClientFactory
+from twisted.web.http import HTTPClient
 
 from scrapy.http import Headers
+from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.responsetypes import responsetypes
 
 
 def _parsed_url_args(parsed):

scrapy/core/engine.py

@@ -15,19 +15,14 @@ from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import (
-    CloseSpider,
-    DontCloseSpider,
-    ScrapyDeprecationWarning,
-)
-from scrapy.http import Response, Request
+from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
+from scrapy.http import Request, Response
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
-from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
+from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.reactor import CallLaterOnce
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/core/http2/agent.py

@@ -10,7 +10,7 @@ from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpoint
 from twisted.web.error import SchemeNotSupported
 
 from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
-from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
+from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
 from scrapy.http.request import Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider

scrapy/core/http2/protocol.py

@@ -9,9 +9,9 @@ from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.errors import ErrorCodes
 from h2.events import (
-    Event,
     ConnectionTerminated,
     DataReceived,
+    Event,
     ResponseReceived,
     SettingsAcknowledged,
     StreamEnded,
@@ -23,7 +23,7 @@ from h2.exceptions import FrameTooLargeError, H2Error
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
-from twisted.internet.protocol import connectionDone, Factory, Protocol
+from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.internet.ssl import Certificate
 from twisted.protocols.policies import TimeoutMixin
 from twisted.python.failure import Failure
@@ -35,7 +35,6 @@ from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/core/http2/stream.py

@@ -1,13 +1,13 @@
 import logging
 from enum import Enum
 from io import BytesIO
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 from urllib.parse import urlparse
-from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 
 from h2.errors import ErrorCodes
 from h2.exceptions import H2Error, ProtocolError, StreamClosedError
 from hpack import HeaderTuple
-from twisted.internet.defer import Deferred, CancelledError
+from twisted.internet.defer import CancelledError, Deferred
 from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed

scrapy/core/scheduler.py

@@ -12,7 +12,6 @@ from scrapy.spiders import Spider
 from scrapy.utils.job import job_dir
 from scrapy.utils.misc import create_instance, load_object
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/core/scraper.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import logging
 from collections import deque
 from typing import (
+    TYPE_CHECKING,
     Any,
     AsyncGenerator,
     AsyncIterable,
@@ -13,7 +14,6 @@ from typing import (
     Iterable,
     Optional,
     Set,
-    TYPE_CHECKING,
     Tuple,
     Union,
 )
@@ -22,7 +22,7 @@ from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
-from scrapy import signals, Spider
+from scrapy import Spider, signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
@@ -34,12 +34,10 @@ from scrapy.utils.defer import (
     parallel,
     parallel_async,
 )
-
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
 
-
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 

scrapy/core/spidermw.py

@@ -28,14 +28,13 @@ from scrapy.middleware import MiddlewareManager
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import (
-    mustbe_deferred,
-    deferred_from_coro,
     deferred_f_from_coro_f,
+    deferred_from_coro,
     maybe_deferred_to_future,
+    mustbe_deferred,
 )
 from scrapy.utils.python import MutableAsyncChain, MutableChain
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/crawler.py

@@ -17,20 +17,20 @@ except ImportError:
 
 from zope.interface.verify import verifyClass
 
-from scrapy import signals, Spider
+from scrapy import Spider, signals
 from scrapy.core.engine import ExecutionEngine
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
-from scrapy.settings import overridden_settings, Settings
+from scrapy.settings import Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
 from scrapy.utils.log import (
+    LogCounterHandler,
     configure_logging,
     get_scrapy_root_handler,
     install_scrapy_root_handler,
     log_reactor_info,
     log_scrapy_info,
-    LogCounterHandler,
 )
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names

scrapy/downloadermiddlewares/ajaxcrawl.py

@@ -1,12 +1,11 @@
-import re
 import logging
+import re
 
 from w3lib import html
 
 from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/downloadermiddlewares/decompression.py

@@ -14,7 +14,6 @@ from warnings import warn
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 
-
 warn(
     "scrapy.downloadermiddlewares.decompression is deprecated",
     ScrapyDeprecationWarning,

scrapy/downloadermiddlewares/httpcache.py

@@ -23,7 +23,6 @@ from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.misc import load_object
 
-
 HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
 
 

scrapy/downloadermiddlewares/httpproxy.py

@@ -1,6 +1,6 @@
 import base64
 from urllib.parse import unquote, urlunparse
-from urllib.request import getproxies, proxy_bypass, _parse_proxy
+from urllib.request import _parse_proxy, getproxies, proxy_bypass
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/downloadermiddlewares/redirect.py

@@ -3,10 +3,10 @@ from urllib.parse import urljoin, urlparse
 
 from w3lib.url import safe_url_string
 
+from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import HtmlResponse
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.response import get_meta_refresh
-from scrapy.exceptions import IgnoreRequest, NotConfigured
 
 logger = logging.getLogger(__name__)
 

scrapy/downloadermiddlewares/retry.py

@@ -9,7 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
-from logging import getLogger, Logger
+from logging import Logger, getLogger
 from typing import Optional, Union
 
 from twisted.internet import defer
@@ -31,7 +31,6 @@ from scrapy.spiders import Spider
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
 
-
 retry_logger = getLogger(__name__)
 
 

scrapy/downloadermiddlewares/robotstxt.py

@@ -7,7 +7,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 import logging
 
 from twisted.internet.defer import Deferred, maybeDeferred
-from scrapy.exceptions import NotConfigured, IgnoreRequest
+
+from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info

scrapy/dupefilters.py

@@ -10,8 +10,7 @@ from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, RequestFingerprinter
-
+from scrapy.utils.request import RequestFingerprinter, referer_str
 
 BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
 

scrapy/exporters.py

@@ -11,14 +11,13 @@ import warnings
 from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator
 
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
-
 __all__ = [
     "BaseItemExporter",
     "PprintItemExporter",

scrapy/extensions/debug.py

@@ -4,11 +4,11 @@ Extensions for debugging Scrapy
 See documentation in docs/topics/extensions.rst
 """
 
-import sys
-import signal
 import logging
-import traceback
+import signal
+import sys
 import threading
+import traceback
 from pdb import Pdb
 
 from scrapy.utils.engine import format_engine_status

scrapy/extensions/feedexport.py

@@ -16,9 +16,9 @@ from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
-from zope.interface import implementer, Interface
+from zope.interface import Interface, implementer
 
-from scrapy import signals, Spider
+from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
@@ -28,7 +28,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import get_func_args, without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/extensions/httpcache.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from time import time
 from weakref import WeakKeyDictionary
 
-from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
+from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
 
 from scrapy.http import Headers, Response
 from scrapy.http.request import Request

scrapy/extensions/logstats.py

@@ -2,8 +2,8 @@ import logging
 
 from twisted.internet import task
 
-from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.exceptions import NotConfigured
 
 logger = logging.getLogger(__name__)
 

scrapy/extensions/memusage.py

@@ -3,11 +3,11 @@ MemoryUsage extension
 
 See documentation in docs/topics/extensions.rst
 """
-import sys
-import socket
 import logging
-from pprint import pformat
+import socket
+import sys
 from importlib import import_module
+from pprint import pformat
 
 from twisted.internet import task
 

scrapy/extensions/statsmailer.py

@@ -5,8 +5,8 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
 """
 
 from scrapy import signals
-from scrapy.mail import MailSender
 from scrapy.exceptions import NotConfigured
+from scrapy.mail import MailSender
 
 
 class StatsMailer:

scrapy/extensions/telnet.py

@@ -4,11 +4,11 @@ Scrapy Telnet Console extension
 See documentation in docs/topics/telnetconsole.rst
 """
 
-import pprint
-import logging
-import traceback
 import binascii
+import logging
 import os
+import pprint
+import traceback
 
 from twisted.internet import protocol
 
@@ -21,12 +21,12 @@ except (ImportError, SyntaxError):
     _TWISTED_CONCH_TRACEBACK = traceback.format_exc()
     TWISTED_CONCH_AVAILABLE = False
 
-from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.trackref import print_live_refs
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.decorators import defers
 from scrapy.utils.engine import print_engine_status
 from scrapy.utils.reactor import listen_tcp
-from scrapy.utils.decorators import defers
+from scrapy.utils.trackref import print_live_refs
 
 logger = logging.getLogger(__name__)
 

scrapy/extensions/throttle.py

@@ -1,7 +1,7 @@
 import logging
 
-from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.exceptions import NotConfigured
 
 logger = logging.getLogger(__name__)
 

scrapy/http/__init__.py

@@ -6,13 +6,11 @@ Request and Response outside this module.
 """
 
 from scrapy.http.headers import Headers
-
 from scrapy.http.request import Request
 from scrapy.http.request.form import FormRequest
-from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.request.json_request import JsonRequest
-
+from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.response import Response
 from scrapy.http.response.html import HtmlResponse
-from scrapy.http.response.xml import XmlResponse
 from scrapy.http.response.text import TextResponse
+from scrapy.http.response.xml import XmlResponse

scrapy/http/cookies.py

@@ -1,11 +1,11 @@
 import re
 import time
-from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy
+from http.cookiejar import CookieJar as _CookieJar
+from http.cookiejar import DefaultCookiePolicy
 
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
-
 # Defined in the http.cookiejar module, but undocumented:
 # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)

scrapy/http/headers.py

@@ -1,6 +1,7 @@
 from collections.abc import Mapping
 
 from w3lib.http import headers_dict_to_raw
+
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
 

scrapy/http/request/__init__.py

@@ -17,7 +17,6 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
-
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 

scrapy/http/request/form.py

@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
-from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit
+from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
 from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
 from parsel.selector import create_root_node
@@ -14,10 +14,9 @@ from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
 from scrapy.http.response.text import TextResponse
-from scrapy.utils.python import to_bytes, is_listlike
+from scrapy.utils.python import is_listlike, to_bytes
 from scrapy.utils.response import get_base_url
 
-
 FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
 
 FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]

scrapy/http/request/rpc.py

@@ -10,7 +10,6 @@ from typing import Optional
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
 
-
 DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 

scrapy/http/response/text.py

@@ -15,8 +15,8 @@ from w3lib.encoding import (
     html_body_declared_encoding,
     html_to_unicode,
     http_content_type_encoding,
-    resolve_encoding,
     read_bom,
+    resolve_encoding,
 )
 from w3lib.html import strip_html5_whitespace
 

scrapy/loader/processors.py

@@ -7,7 +7,6 @@ from itemloaders import processors
 
 from scrapy.utils.deprecate import create_deprecated_class
 
-
 MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
 
 Compose = create_deprecated_class("Compose", processors.Compose)

scrapy/logformatter.py

@@ -1,5 +1,5 @@
-import os
 import logging
+import os
 
 from twisted.python.failure import Failure
 

scrapy/mail.py

@@ -12,14 +12,13 @@ from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
 
-from twisted.python.versions import Version
-from twisted.internet import defer, ssl
 from twisted import version as twisted_version
+from twisted.internet import defer, ssl
+from twisted.python.versions import Version
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/middleware.py

@@ -8,8 +8,8 @@ from twisted.internet.defer import Deferred
 from scrapy import Spider
 from scrapy.exceptions import NotConfigured
 from scrapy.settings import Settings
+from scrapy.utils.defer import process_chain, process_parallel
 from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.defer import process_parallel, process_chain
 
 logger = logging.getLogger(__name__)
 

scrapy/pipelines/files.py

@@ -32,7 +32,6 @@ from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/pipelines/media.py

@@ -9,10 +9,10 @@ from twisted.python.failure import Failure
 
 from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
-from scrapy.utils.defer import mustbe_deferred, defer_result
+from scrapy.utils.defer import defer_result, mustbe_deferred
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.misc import arg_to_iter
 
 logger = logging.getLogger(__name__)
 

scrapy/pqueues.py

@@ -3,7 +3,6 @@ import logging
 
 from scrapy.utils.misc import create_instance
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/resolver.py

@@ -1,8 +1,8 @@
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
 from twisted.internet.interfaces import (
-    IHostResolution,
     IHostnameResolver,
+    IHostResolution,
     IResolutionReceiver,
     IResolverSimple,
 )
@@ -10,7 +10,6 @@ from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
 
-
 # TODO: cache misses
 dnscache = LocalCache(10000)
 

scrapy/responsetypes.py

@@ -2,9 +2,9 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
+from io import StringIO
 from mimetypes import MimeTypes
 from pkgutil import get_data
-from io import StringIO
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object

scrapy/robotstxt.py

@@ -1,5 +1,5 @@
-import sys
 import logging
+import sys
 from abc import ABCMeta, abstractmethod
 
 from scrapy.utils.python import to_unicode

scrapy/selector/unified.py

@@ -3,10 +3,10 @@ XPath selectors based on lxml
 """
 
 from parsel import Selector as _ParselSelector
-from scrapy.utils.trackref import object_ref
-from scrapy.utils.python import to_bytes
-from scrapy.http import HtmlResponse, XmlResponse
 
+from scrapy.http import HtmlResponse, XmlResponse
+from scrapy.utils.python import to_bytes
+from scrapy.utils.trackref import object_ref
 
 __all__ = ["Selector", "SelectorList"]
 

scrapy/settings/__init__.py

@@ -1,12 +1,11 @@
-import json
 import copy
+import json
 from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
 from scrapy.settings import default_settings
 
-
 SETTINGS_PRIORITIES = {
     "default": 0,
     "command": 10,

scrapy/shell.py

@@ -7,7 +7,7 @@ import os
 import signal
 
 from itemadapter import is_item
-from twisted.internet import threads, defer
+from twisted.internet import defer, threads
 from twisted.python import threadable
 from w3lib.url import any_to_uri
 
@@ -20,11 +20,8 @@ from scrapy.utils.conf import get_config
 from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 from scrapy.utils.response import open_in_browser
-from scrapy.utils.reactor import (
-    is_asyncio_reactor_installed,
-    set_asyncio_event_loop,
-)
 
 
 class Shell:

scrapy/signalmanager.py

@@ -1,4 +1,5 @@
 from pydispatch import dispatcher
+
 from scrapy.utils import signal as _signal
 
 

scrapy/spidermiddlewares/offsite.py

@@ -3,8 +3,8 @@ Offsite Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
-import re
 import logging
+import re
 import warnings
 
 from scrapy import signals

scrapy/spidermiddlewares/referer.py

@@ -15,7 +15,6 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
 
-
 LOCAL_SCHEMES = (
     "about",
     "blob",

scrapy/spidermiddlewares/urllength.py

@@ -6,8 +6,8 @@ See documentation in docs/topics/spider-middleware.rst
 
 import logging
 
-from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Request
 
 logger = logging.getLogger(__name__)
 

scrapy/spiders/__init__.py

@@ -97,5 +97,5 @@ class Spider(object_ref):
 
 # Top-level imports
 from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider
 from scrapy.spiders.sitemap import SitemapSpider

scrapy/spiders/crawl.py

@@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst
 import copy
 from typing import AsyncIterable, Awaitable, Sequence
 
-from scrapy.http import Request, Response, HtmlResponse
+from scrapy.http import HtmlResponse, Request, Response
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.utils.asyncgen import collect_asyncgen

scrapy/spiders/feed.py

@@ -4,11 +4,11 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
-from scrapy.spiders import Spider
-from scrapy.utils.iterators import xmliter, csviter
-from scrapy.utils.spider import iterate_spider_output
-from scrapy.selector import Selector
 from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.selector import Selector
+from scrapy.spiders import Spider
+from scrapy.utils.iterators import csviter, xmliter
+from scrapy.utils.spider import iterate_spider_output
 
 
 class XMLFeedSpider(Spider):

scrapy/spiders/sitemap.py

@@ -1,11 +1,10 @@
-import re
 import logging
+import re
 
-from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
-from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
+from scrapy.spiders import Spider
 from scrapy.utils.gz import gunzip, gzip_magic_number
-
+from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 
 logger = logging.getLogger(__name__)
 

scrapy/statscollectors.py

@@ -1,8 +1,8 @@
 """
 Scrapy extension for collecting scraping stats
 """
-import pprint
 import logging
+import pprint
 
 logger = logging.getLogger(__name__)
 

scrapy/utils/benchserver.py

@@ -1,8 +1,8 @@
 import random
 from urllib.parse import urlencode
 
-from twisted.web.server import Site
 from twisted.web.resource import Resource
+from twisted.web.server import Site
 
 
 class Root(Resource):

scrapy/utils/conf.py

@@ -8,7 +8,6 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
-
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values

scrapy/utils/curl.py

@@ -1,7 +1,7 @@
 import argparse
 import warnings
-from shlex import split
 from http.cookies import SimpleCookie
+from shlex import split
 from urllib.parse import urlparse
 
 from w3lib.http import basic_auth_header

scrapy/utils/defer.py

@@ -26,7 +26,7 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop
+from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed
 
 
 def defer_fail(_failure: Failure) -> Deferred:

scrapy/utils/deprecate.py

@@ -1,8 +1,9 @@
 """Some helpers for deprecation messages"""
 
-import warnings
 import inspect
+import warnings
 from typing import List, Tuple
+
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 

scrapy/utils/display.py

@@ -6,6 +6,7 @@ import ctypes
 import platform
 import sys
 from pprint import pformat as pformat_
+
 from packaging.version import Version as parse_version
 
 

scrapy/utils/ftp.py

@@ -1,5 +1,5 @@
 import posixpath
-from ftplib import error_perm, FTP
+from ftplib import FTP, error_perm
 from posixpath import dirname
 
 

scrapy/utils/httpobj.py

@@ -1,12 +1,11 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
 from typing import Union
-from urllib.parse import urlparse, ParseResult
+from urllib.parse import ParseResult, urlparse
 from weakref import WeakKeyDictionary
 
 from scrapy.http import Request, Response
 
-
 _urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = (
     WeakKeyDictionary()
 )

scrapy/utils/iterators.py

@@ -3,11 +3,10 @@ import logging
 import re
 from io import StringIO
 
-from scrapy.http import TextResponse, Response
+from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/utils/log.py

@@ -11,7 +11,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.versions import scrapy_components_versions
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/utils/misc.py

@@ -1,23 +1,22 @@
 """Helper functions which don't fit anywhere else"""
 import ast
+import hashlib
 import inspect
 import os
 import re
-import hashlib
 import warnings
 from collections import deque
 from contextlib import contextmanager
+from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from functools import partial
 
 from w3lib.html import replace_entities
 
-from scrapy.utils.datatypes import LocalWeakReferencedCache
-from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import Item
+from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
-
+from scrapy.utils.python import flatten, to_unicode
 
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 

scrapy/utils/ossignal.py

@@ -1,6 +1,5 @@
 import signal
 
-
 signal_names = {}
 for signame in dir(signal):
     if signame.startswith("SIG") and not signame.startswith("SIG_"):

scrapy/utils/project.py

@@ -1,13 +1,11 @@
 import os
 import warnings
-
 from importlib import import_module
 from pathlib import Path
 
-from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
-from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
-
+from scrapy.settings import Settings
+from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 
 ENVVAR = "SCRAPY_SETTINGS_MODULE"
 DATADIR_CFG_SECTION = "datadir"

scrapy/utils/reqser.py

@@ -5,7 +5,6 @@ import scrapy
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.request import request_from_dict as _from_dict
 
-
 warnings.warn(
     (
         "Module scrapy.utils.reqser is deprecated, please use request.to_dict method"

scrapy/utils/request.py

@@ -19,7 +19,6 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
-
 _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
 _deprecated_fingerprint_cache = WeakKeyDictionary()
 

scrapy/utils/response.py

@@ -11,12 +11,11 @@ from weakref import WeakKeyDictionary
 
 from twisted.web import http
 from w3lib import html
+
 import scrapy
 from scrapy.http.response import Response
-
-
-from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.utils.decorators import deprecated
+from scrapy.utils.python import to_bytes, to_unicode
 
 _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
 

scrapy/utils/serialize.py

@@ -1,8 +1,8 @@
-import json
 import datetime
 import decimal
+import json
 
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
 from twisted.internet import defer
 
 from scrapy.http import Request, Response

scrapy/utils/signal.py

@@ -2,9 +2,6 @@
 import collections.abc
 import logging
 
-from twisted.internet.defer import DeferredList, Deferred
-from twisted.python.failure import Failure
-
 from pydispatch.dispatcher import (
     Anonymous,
     Any,
@@ -13,12 +10,13 @@ from pydispatch.dispatcher import (
     liveReceivers,
 )
 from pydispatch.robustapply import robustApply
+from twisted.internet.defer import Deferred, DeferredList
+from twisted.python.failure import Failure
 
 from scrapy.exceptions import StopDownload
 from scrapy.utils.defer import maybeDeferred_coro
 from scrapy.utils.log import failure_to_exc_info
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/utils/spider.py

@@ -5,7 +5,6 @@ from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/utils/ssl.py

@@ -1,5 +1,5 @@
-import OpenSSL.SSL
 import OpenSSL._util as pyOpenSSLutil
+import OpenSSL.SSL
 
 from scrapy.utils.python import to_unicode