Add flake8-type-checking rules to ruff.

pyproject.toml

@@ -244,6 +244,8 @@ extend-select = [
     "D",
     # flake8-debugger
     "T10",
+    # flake8-type-checking
+    "TC",
 ]
 ignore = [
     # Assigning to `os.environ` doesn't clear the environment.

scrapy/commands/bench.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import subprocess  # nosec
 import sys
 import time
@@ -13,6 +12,7 @@ from scrapy.http import Response, TextResponse
 from scrapy.linkextractors import LinkExtractor
 
 if TYPE_CHECKING:
+    import argparse
     from collections.abc import Iterable
 
     from scrapy import Request

scrapy/commands/fetch.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
 
 from w3lib.url import is_url
 
-from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.http import Request, Response
@@ -15,6 +14,8 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 if TYPE_CHECKING:
     from argparse import ArgumentParser, Namespace
 
+    from scrapy import Spider
+
 
 class Command(ScrapyCommand):
     requires_project = False

scrapy/commands/genspider.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
-import argparse
 import os
 import shutil
 import string
 from importlib import import_module
 from pathlib import Path
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urlparse
 
 import scrapy
@@ -14,6 +13,9 @@ from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
+if TYPE_CHECKING:
+    import argparse
+
 
 def sanitize_module_name(module_name: str) -> str:
     """Sanitize the given module name, by replacing dashes and points

scrapy/commands/parse.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import functools
 import inspect
 import json
@@ -22,6 +21,7 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.spider import spidercls_for_request
 
 if TYPE_CHECKING:
+    import argparse
     from collections.abc import AsyncGenerator, Coroutine, Iterable
 
     from twisted.python.failure import Failure

scrapy/commands/runspider.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import sys
 from importlib import import_module
 from pathlib import Path
@@ -11,6 +10,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.spider import iter_spider_classes
 
 if TYPE_CHECKING:
+    import argparse
     from os import PathLike
     from types import ModuleType
 

scrapy/commands/shell.py

@@ -9,7 +9,6 @@ from __future__ import annotations
 from threading import Thread
 from typing import TYPE_CHECKING, Any
 
-from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
@@ -19,6 +18,8 @@ from scrapy.utils.url import guess_scheme
 if TYPE_CHECKING:
     from argparse import ArgumentParser, Namespace
 
+    from scrapy import Spider
+
 
 class Command(ScrapyCommand):
     requires_project = False

scrapy/commands/startproject.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import argparse
 import os
 import re
 import string
@@ -8,12 +7,16 @@ from importlib.util import find_spec
 from pathlib import Path
 from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
+from typing import TYPE_CHECKING
 
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
+if TYPE_CHECKING:
+    import argparse
+
 TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (
     ("scrapy.cfg",),
     ("${project_name}", "settings.py.tmpl"),

scrapy/core/downloader/__init__.py

@@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.resolver import dnscache
-from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 
@@ -23,6 +22,7 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
     from scrapy.http import Response
     from scrapy.settings import BaseSettings
+    from scrapy.signalmanager import SignalManager
 
 
 _T = TypeVar("_T")

scrapy/core/downloader/handlers/__init__.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import logging
-from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, Protocol, cast
 
 from twisted.internet import defer
@@ -15,7 +14,7 @@ from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Callable, Generator
 
     from twisted.internet.defer import Deferred
 

scrapy/core/engine.py

@@ -17,13 +17,9 @@ from twisted.internet.task import LoopingCall
 from twisted.python.failure import Failure
 
 from scrapy import signals
-from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper, _HandleOutputDeferred
 from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.logformatter import LogFormatter
-from scrapy.settings import Settings
-from scrapy.signalmanager import SignalManager
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.reactor import CallLaterOnce
@@ -31,9 +27,12 @@ from scrapy.utils.reactor import CallLaterOnce
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator, Iterable, Iterator
 
+    from scrapy.core.downloader import Downloader
     from scrapy.core.scheduler import BaseScheduler
     from scrapy.crawler import Crawler
-    from scrapy.settings import BaseSettings
+    from scrapy.logformatter import LogFormatter
+    from scrapy.settings import BaseSettings, Settings
+    from scrapy.signalmanager import SignalManager
     from scrapy.spiders import Spider
 
 

scrapy/core/scraper.py

@@ -16,9 +16,6 @@ from scrapy import Spider, signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.logformatter import LogFormatter
-from scrapy.pipelines import ItemPipelineManager
-from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import (
     aiter_errback,
     defer_fail,
@@ -35,6 +32,9 @@ if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
 
     from scrapy.crawler import Crawler
+    from scrapy.logformatter import LogFormatter
+    from scrapy.pipelines import ItemPipelineManager
+    from scrapy.signalmanager import SignalManager
 
 
 logger = logging.getLogger(__name__)

scrapy/crawler.py

@@ -18,10 +18,8 @@ from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
 from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
-from scrapy.logformatter import LogFormatter
 from scrapy.settings import BaseSettings, Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.log import (
     LogCounterHandler,
     configure_logging,
@@ -42,7 +40,9 @@ from scrapy.utils.reactor import (
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
 
+    from scrapy.logformatter import LogFormatter
     from scrapy.spiderloader import SpiderLoader
+    from scrapy.statscollectors import StatsCollector
     from scrapy.utils.request import RequestFingerprinter
 
 

scrapy/exporters.py

@@ -10,8 +10,7 @@ import pickle  # nosec
 import pprint
 from collections.abc import Callable, Iterable, Mapping
 from io import BytesIO, TextIOWrapper
-from json import JSONEncoder
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from xml.sax.saxutils import XMLGenerator  # nosec
 from xml.sax.xmlreader import AttributesImpl  # nosec
 
@@ -21,6 +20,9 @@ from scrapy.item import Field, Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
+if TYPE_CHECKING:
+    from json import JSONEncoder
+
 __all__ = [
     "BaseItemExporter",
     "PprintItemExporter",

scrapy/extensions/feedexport.py

@@ -25,7 +25,6 @@ from zope.interface import Interface, implementer
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.settings import Settings
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.ftp import ftp_store_file
@@ -44,7 +43,7 @@ if TYPE_CHECKING:
 
     from scrapy.crawler import Crawler
     from scrapy.exporters import BaseItemExporter
-    from scrapy.settings import BaseSettings
+    from scrapy.settings import BaseSettings, Settings
 
 
 logger = logging.getLogger(__name__)

scrapy/extensions/httpcache.py

@@ -2,13 +2,11 @@ from __future__ import annotations
 
 import gzip
 import logging
-import os
 import pickle  # nosec
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
 from time import time
-from types import ModuleType
 from typing import IO, TYPE_CHECKING, Any, cast
 from weakref import WeakKeyDictionary
 
@@ -19,10 +17,11 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import RequestFingerprinter
 
 if TYPE_CHECKING:
+    import os
     from collections.abc import Callable
+    from types import ModuleType
 
     # typing.Concatenate requires Python 3.10
     from typing_extensions import Concatenate
@@ -30,6 +29,7 @@ if TYPE_CHECKING:
     from scrapy.http.request import Request
     from scrapy.settings import BaseSettings
     from scrapy.spiders import Spider
+    from scrapy.utils.request import RequestFingerprinter
 
 
 logger = logging.getLogger(__name__)

scrapy/extensions/periodic_log.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 from datetime import datetime, timezone
-from json import JSONEncoder
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet import task
@@ -13,6 +12,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
+    from json import JSONEncoder
+
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler

scrapy/extensions/telnet.py

@@ -13,7 +13,6 @@ import pprint
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet import protocol
-from twisted.internet.tcp import Port
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -24,6 +23,7 @@ from scrapy.utils.trackref import print_live_refs
 
 if TYPE_CHECKING:
     from twisted.conch import telnet
+    from twisted.internet.tcp import Port
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self

scrapy/http/request/__init__.py

@@ -21,7 +21,6 @@ from typing import (
 
 from w3lib.url import safe_url_string
 
-import scrapy
 from scrapy.http.headers import Headers
 from scrapy.utils.curl import curl_to_request_kwargs
 from scrapy.utils.python import to_bytes
@@ -37,6 +36,7 @@ if TYPE_CHECKING:
     # typing.NotRequired and typing.Self require Python 3.11
     from typing_extensions import Concatenate, NotRequired, Self
 
+    from scrapy import Spider
     from scrapy.http import Response
 
     CallbackT = Callable[Concatenate[Response, ...], Any]
@@ -252,7 +252,7 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
+    def to_dict(self, *, spider: Spider | None = None) -> dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.

scrapy/pqueues.py

@@ -5,7 +5,6 @@ import logging
 from typing import TYPE_CHECKING, Protocol, cast
 
 from scrapy import Request
-from scrapy.core.downloader import Downloader
 from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
@@ -14,6 +13,7 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.core.downloader import Downloader
     from scrapy.crawler import Crawler
 
 logger = logging.getLogger(__name__)

scrapy/utils/conf.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import numbers
 import os
 import sys
-from collections.abc import Iterable
 from configparser import ConfigParser
 from operator import itemgetter
 from pathlib import Path
@@ -15,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
-    from collections.abc import Collection, Mapping, MutableMapping
+    from collections.abc import Collection, Iterable, Mapping, MutableMapping
 
 
 def build_component_list(

scrapy/utils/reactor.py

@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar
 from warnings import catch_warnings, filterwarnings
 
 from twisted.internet import asyncioreactor, error
-from twisted.internet.base import DelayedCall
 
 from scrapy.utils.misc import load_object
 
@@ -15,6 +14,7 @@ if TYPE_CHECKING:
     from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
     from collections.abc import Callable
 
+    from twisted.internet.base import DelayedCall
     from twisted.internet.protocol import ServerFactory
     from twisted.internet.tcp import Port
 

scrapy/utils/test.py

@@ -16,7 +16,6 @@ from unittest import TestCase, mock
 from twisted.trial.unittest import SkipTest
 
 from scrapy import Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 
@@ -26,6 +25,8 @@ if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
     from twisted.web.client import Response as TxResponse
 
+    from scrapy.crawler import Crawler
+
 
 _T = TypeVar("_T")