Merge pull request #6576 from wRAR/ruff-rules-1

Add flake8 plugin rules to ruff

.flake8

@@ -1,82 +0,0 @@
-[flake8]
-
-max-line-length = 119
-extend-select = TC, TC1
-ignore =
-    # black disagrees with flake8 about these
-    E203, E501, E701, E704, W503
-
-    # Assigning to `os.environ` doesn't clear the environment.
-    B003
-    # Do not use mutable data structures for argument defaults.
-    B006
-    # Loop control variable not used within the loop body.
-    B007
-    # Do not perform function calls in argument defaults.
-    B008
-    # return/continue/break inside finally blocks cause exceptions to be
-    # silenced.
-    B012
-    # Star-arg unpacking after a keyword argument is strongly discouraged
-    B026
-    # No explicit stacklevel argument found.
-    B028
-
-    # docstring does contain unindexed parameters
-    P102
-    # other string does contain unindexed parameters
-    P103
-
-    # Missing docstring in public module
-    D100
-    # Missing docstring in public class
-    D101
-    # Missing docstring in public method
-    D102
-    # Missing docstring in public function
-    D103
-    # Missing docstring in public package
-    D104
-    # Missing docstring in magic method
-    D105
-    # Missing docstring in public nested class
-    D106
-    # Missing docstring in __init__
-    D107
-    # One-line docstring should fit on one line with quotes
-    D200
-    # No blank lines allowed after function docstring
-    D202
-    # 1 blank line required between summary line and description
-    D205
-    # Multi-line docstring closing quotes should be on a separate line
-    D209
-    # First line should end with a period
-    D400
-    # First line should be in imperative mood; try rephrasing
-    D401
-    # First line should not be the function's "signature"
-    D402
-    # First word of the first line should be properly capitalized
-    D403
-
-    # Annotation in typing.cast() should be a string literal
-    TC006
-exclude =
-    docs/conf.py
-
-per-file-ignores =
-# Exclude files that are meant to provide top-level imports
-# E402: Module level import not at top of file
-# F401: Module imported but unused
-    scrapy/__init__.py:E402
-    scrapy/core/downloader/handlers/http.py:F401
-    scrapy/http/__init__.py:F401
-    scrapy/linkextractors/__init__.py:E402,F401
-    scrapy/selector/__init__.py:F401
-    scrapy/spiders/__init__.py:E402,F401
-    tests/CrawlerRunner/change_reactor.py:E402
-
-    # Issues pending a review:
-    scrapy/utils/url.py:F403,F405
-    tests/test_loader.py:E741

.pre-commit-config.yaml

@@ -9,17 +9,6 @@ repos:
   - id: bandit
     args: ["-c", "pyproject.toml"]
     additional_dependencies: ["bandit[toml]"]
-- repo: https://github.com/PyCQA/flake8
-  rev: 7.1.0
-  hooks:
-  - id: flake8
-    additional_dependencies:
-    - flake8-bugbear
-    - flake8-comprehensions
-    - flake8-debugger
-    - flake8-docstrings
-    - flake8-string-format
-    - flake8-type-checking
 - repo: https://github.com/psf/black.git
   rev: 24.4.2
   hooks:

pyproject.toml

@@ -236,8 +236,66 @@ filterwarnings = []
 
 [tool.ruff.lint]
 extend-select = [
+    # flake8-bugbear
+    "B",
+    # flake8-comprehensions
+    "C4",
+    # pydocstyle
+    "D",
+    # flake8-debugger
+    "T10",
+    # flake8-type-checking
+    "TC",
 ]
 ignore = [
+    # Assigning to `os.environ` doesn't clear the environment.
+    "B003",
+    # Do not use mutable data structures for argument defaults.
+    "B006",
+    # Loop control variable not used within the loop body.
+    "B007",
+    # Do not perform function calls in argument defaults.
+    "B008",
+    # Star-arg unpacking after a keyword argument is strongly discouraged.
+    "B026",
+    # Found useless expression.
+    "B018",
+    # No explicit stacklevel argument found.
+    "B028",
+    # Within an `except` clause, raise exceptions with `raise ... from`
+    "B904",
+    # Missing docstring in public module
+    "D100",
+    # Missing docstring in public class
+    "D101",
+    # Missing docstring in public method
+    "D102",
+    # Missing docstring in public function
+    "D103",
+    # Missing docstring in public package
+    "D104",
+    # Missing docstring in magic method
+    "D105",
+    # Missing docstring in public nested class
+    "D106",
+    # Missing docstring in __init__
+    "D107",
+    # One-line docstring should fit on one line with quotes
+    "D200",
+    # No blank lines allowed after function docstring
+    "D202",
+    # 1 blank line required between summary line and description
+    "D205",
+    # Multi-line docstring closing quotes should be on a separate line
+    "D209",
+    # First line should end with a period
+    "D400",
+    # First line should be in imperative mood; try rephrasing
+    "D401",
+    # First line should not be the function's "signature"
+    "D402",
+    # First word of the first line should be properly capitalized
+    "D403",
 ]
 
 [tool.ruff.lint.per-file-ignores]
@@ -252,5 +310,7 @@ ignore = [
 # Issues pending a review:
 "docs/conf.py" = ["E402"]
 "scrapy/utils/url.py" = ["F403", "F405"]
-"tests/CrawlerRunner/change_reactor.py" = ["E402"]
 "tests/test_loader.py" = ["E741"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"

scrapy/commands/bench.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import subprocess  # nosec
 import sys
 import time
@@ -13,6 +12,7 @@ from scrapy.http import Response, TextResponse
 from scrapy.linkextractors import LinkExtractor
 
 if TYPE_CHECKING:
+    import argparse
     from collections.abc import Iterable
 
     from scrapy import Request

scrapy/commands/fetch.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
 
 from w3lib.url import is_url
 
-from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.http import Request, Response
@@ -15,6 +14,8 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 if TYPE_CHECKING:
     from argparse import ArgumentParser, Namespace
 
+    from scrapy import Spider
+
 
 class Command(ScrapyCommand):
     requires_project = False

scrapy/commands/genspider.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
-import argparse
 import os
 import shutil
 import string
 from importlib import import_module
 from pathlib import Path
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urlparse
 
 import scrapy
@@ -14,6 +13,9 @@ from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
+if TYPE_CHECKING:
+    import argparse
+
 
 def sanitize_module_name(module_name: str) -> str:
     """Sanitize the given module name, by replacing dashes and points

scrapy/commands/parse.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import functools
 import inspect
 import json
@@ -22,6 +21,7 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.spider import spidercls_for_request
 
 if TYPE_CHECKING:
+    import argparse
     from collections.abc import AsyncGenerator, Coroutine, Iterable
 
     from twisted.python.failure import Failure

scrapy/commands/runspider.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import sys
 from importlib import import_module
 from pathlib import Path
@@ -11,6 +10,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.spider import iter_spider_classes
 
 if TYPE_CHECKING:
+    import argparse
     from os import PathLike
     from types import ModuleType
 

scrapy/commands/shell.py

@@ -9,7 +9,6 @@ from __future__ import annotations
 from threading import Thread
 from typing import TYPE_CHECKING, Any
 
-from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
@@ -19,6 +18,8 @@ from scrapy.utils.url import guess_scheme
 if TYPE_CHECKING:
     from argparse import ArgumentParser, Namespace
 
+    from scrapy import Spider
+
 
 class Command(ScrapyCommand):
     requires_project = False

scrapy/commands/startproject.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import os
 import re
 import string
@@ -8,12 +7,16 @@ from importlib.util import find_spec
 from pathlib import Path
 from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
+from typing import TYPE_CHECKING
 
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
+if TYPE_CHECKING:
+    import argparse
+
 TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (
     ("scrapy.cfg",),
     ("${project_name}", "settings.py.tmpl"),

scrapy/core/downloader/__init__.py

@@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.resolver import dnscache
-from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 
@@ -23,6 +22,7 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
     from scrapy.http import Response
     from scrapy.settings import BaseSettings
+    from scrapy.signalmanager import SignalManager
 
 
 _T = TypeVar("_T")

scrapy/core/downloader/handlers/__init__.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import logging
-from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, Protocol, cast
 
 from twisted.internet import defer
@@ -15,7 +14,7 @@ from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Callable, Generator
 
     from twisted.internet.defer import Deferred
 

scrapy/core/engine.py

@@ -17,13 +17,9 @@ from twisted.internet.task import LoopingCall
 from twisted.python.failure import Failure
 
 from scrapy import signals
-from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper, _HandleOutputDeferred
 from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.logformatter import LogFormatter
-from scrapy.settings import Settings
-from scrapy.signalmanager import SignalManager
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.reactor import CallLaterOnce
@@ -31,9 +27,12 @@ from scrapy.utils.reactor import CallLaterOnce
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator, Iterable, Iterator
 
+    from scrapy.core.downloader import Downloader
     from scrapy.core.scheduler import BaseScheduler
     from scrapy.crawler import Crawler
-    from scrapy.settings import BaseSettings
+    from scrapy.logformatter import LogFormatter
+    from scrapy.settings import BaseSettings, Settings
+    from scrapy.signalmanager import SignalManager
     from scrapy.spiders import Spider
 
 

scrapy/core/scraper.py

@@ -16,9 +16,6 @@ from scrapy import Spider, signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.logformatter import LogFormatter
-from scrapy.pipelines import ItemPipelineManager
-from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import (
     aiter_errback,
     defer_fail,
@@ -35,6 +32,9 @@ if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
 
     from scrapy.crawler import Crawler
+    from scrapy.logformatter import LogFormatter
+    from scrapy.pipelines import ItemPipelineManager
+    from scrapy.signalmanager import SignalManager
 
 
 logger = logging.getLogger(__name__)

scrapy/crawler.py

@@ -18,10 +18,8 @@ from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
 from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
-from scrapy.logformatter import LogFormatter
 from scrapy.settings import BaseSettings, Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.log import (
     LogCounterHandler,
     configure_logging,
@@ -42,7 +40,9 @@ from scrapy.utils.reactor import (
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
 
+    from scrapy.logformatter import LogFormatter
     from scrapy.spiderloader import SpiderLoader
+    from scrapy.statscollectors import StatsCollector
     from scrapy.utils.request import RequestFingerprinter
 
 

scrapy/exporters.py

@@ -10,8 +10,7 @@ import pickle  # nosec
 import pprint
 from collections.abc import Callable, Iterable, Mapping
 from io import BytesIO, TextIOWrapper
-from json import JSONEncoder
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from xml.sax.saxutils import XMLGenerator  # nosec
 from xml.sax.xmlreader import AttributesImpl  # nosec
 
@@ -21,6 +20,9 @@ from scrapy.item import Field, Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
+if TYPE_CHECKING:
+    from json import JSONEncoder
+
 __all__ = [
     "BaseItemExporter",
     "PprintItemExporter",

scrapy/extensions/feedexport.py

@@ -25,7 +25,6 @@ from zope.interface import Interface, implementer
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.settings import Settings
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.ftp import ftp_store_file
@@ -44,7 +43,7 @@ if TYPE_CHECKING:
 
     from scrapy.crawler import Crawler
     from scrapy.exporters import BaseItemExporter
-    from scrapy.settings import BaseSettings
+    from scrapy.settings import BaseSettings, Settings
 
 
 logger = logging.getLogger(__name__)

scrapy/extensions/httpcache.py

@@ -2,13 +2,11 @@ from __future__ import annotations
 
 import gzip
 import logging
-import os
 import pickle  # nosec
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
 from time import time
-from types import ModuleType
 from typing import IO, TYPE_CHECKING, Any, cast
 from weakref import WeakKeyDictionary
 
@@ -19,10 +17,11 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import RequestFingerprinter
 
 if TYPE_CHECKING:
+    import os
     from collections.abc import Callable
+    from types import ModuleType
 
     # typing.Concatenate requires Python 3.10
     from typing_extensions import Concatenate
@@ -30,6 +29,7 @@ if TYPE_CHECKING:
     from scrapy.http.request import Request
     from scrapy.settings import BaseSettings
     from scrapy.spiders import Spider
+    from scrapy.utils.request import RequestFingerprinter
 
 
 logger = logging.getLogger(__name__)

scrapy/extensions/periodic_log.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 from datetime import datetime, timezone
-from json import JSONEncoder
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet import task
@@ -13,6 +12,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
+    from json import JSONEncoder
+
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler

scrapy/extensions/telnet.py

@@ -13,7 +13,6 @@ import pprint
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet import protocol
-from twisted.internet.tcp import Port
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -24,6 +23,7 @@ from scrapy.utils.trackref import print_live_refs
 
 if TYPE_CHECKING:
     from twisted.conch import telnet
+    from twisted.internet.tcp import Port
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self

scrapy/http/request/__init__.py

@@ -21,7 +21,8 @@ from typing import (
 
 from w3lib.url import safe_url_string
 
-import scrapy
+# a workaround for the docs "more than one target found" problem
+import scrapy  # noqa: TC001
 from scrapy.http.headers import Headers
 from scrapy.utils.curl import curl_to_request_kwargs
 from scrapy.utils.python import to_bytes

scrapy/pqueues.py

@@ -5,7 +5,6 @@ import logging
 from typing import TYPE_CHECKING, Protocol, cast
 
 from scrapy import Request
-from scrapy.core.downloader import Downloader
 from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
@@ -14,6 +13,7 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.core.downloader import Downloader
     from scrapy.crawler import Crawler
 
 logger = logging.getLogger(__name__)

scrapy/utils/conf.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import numbers
 import os
 import sys
-from collections.abc import Iterable
 from configparser import ConfigParser
 from operator import itemgetter
 from pathlib import Path
@@ -15,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
-    from collections.abc import Collection, Mapping, MutableMapping
+    from collections.abc import Collection, Iterable, Mapping, MutableMapping
 
 
 def build_component_list(

scrapy/utils/reactor.py

@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar
 from warnings import catch_warnings, filterwarnings
 
 from twisted.internet import asyncioreactor, error
-from twisted.internet.base import DelayedCall
 
 from scrapy.utils.misc import load_object
 
@@ -15,6 +14,7 @@ if TYPE_CHECKING:
     from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
     from collections.abc import Callable
 
+    from twisted.internet.base import DelayedCall
     from twisted.internet.protocol import ServerFactory
     from twisted.internet.tcp import Port
 

scrapy/utils/test.py

@@ -16,7 +16,6 @@ from unittest import TestCase, mock
 from twisted.trial.unittest import SkipTest
 
 from scrapy import Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 
@@ -26,6 +25,8 @@ if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
     from twisted.web.client import Response as TxResponse
 
+    from scrapy.crawler import Crawler
+
 
 _T = TypeVar("_T")
 

tests/CrawlerRunner/change_reactor.py

@@ -17,7 +17,7 @@ class NoRequestsSpider(Spider):
 configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"})
 
 
-from scrapy.utils.reactor import install_reactor
+from scrapy.utils.reactor import install_reactor  # noqa: E402
 
 install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
 
@@ -25,7 +25,7 @@ runner = CrawlerRunner()
 
 d = runner.crawl(NoRequestsSpider)
 
-from twisted.internet import reactor
+from twisted.internet import reactor  # noqa: E402
 
 d.addBoth(callback=lambda _: reactor.stop())
 reactor.run()

tests/test_feedexport.py

@@ -473,7 +473,7 @@ class S3FeedStorageTest(unittest.TestCase):
 class GCSFeedStorageTest(unittest.TestCase):
     def test_parse_settings(self):
         try:
-            from google.cloud.storage import Client  # noqa
+            from google.cloud.storage import Client  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
@@ -487,7 +487,7 @@ class GCSFeedStorageTest(unittest.TestCase):
 
     def test_parse_empty_acl(self):
         try:
-            from google.cloud.storage import Client  # noqa
+            from google.cloud.storage import Client  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
@@ -504,7 +504,7 @@ class GCSFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         try:
-            from google.cloud.storage import Client  # noqa
+            from google.cloud.storage import Client  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 

tests/test_item.py

@@ -273,9 +273,7 @@ class ItemMetaTest(unittest.TestCase):
                 def f(self):
                     # For rationale of this see:
                     # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
-                    return (
-                        __class__  # noqa  https://github.com/scrapy/scrapy/issues/2836
-                    )
+                    return __class__
 
             MyItem()