From d47f142d0ffff0e8f87eee67c4c0e7edbc71613e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 1 Dec 2024 12:28:57 +0500 Subject: [PATCH] Add flake8-type-checking rules to ruff. --- pyproject.toml | 2 ++ scrapy/commands/bench.py | 2 +- scrapy/commands/fetch.py | 3 ++- scrapy/commands/genspider.py | 6 ++++-- scrapy/commands/parse.py | 2 +- scrapy/commands/runspider.py | 2 +- scrapy/commands/shell.py | 3 ++- scrapy/commands/startproject.py | 5 ++++- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/__init__.py | 3 +-- scrapy/core/engine.py | 9 ++++----- scrapy/core/scraper.py | 6 +++--- scrapy/crawler.py | 4 ++-- scrapy/exporters.py | 6 ++++-- scrapy/extensions/feedexport.py | 3 +-- scrapy/extensions/httpcache.py | 6 +++--- scrapy/extensions/periodic_log.py | 3 ++- scrapy/extensions/telnet.py | 2 +- scrapy/http/request/__init__.py | 4 ++-- scrapy/pqueues.py | 2 +- scrapy/utils/conf.py | 3 +-- scrapy/utils/reactor.py | 2 +- scrapy/utils/test.py | 3 ++- 23 files changed, 46 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41ba77094..4d20e5c1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,6 +244,8 @@ extend-select = [ "D", # flake8-debugger "T10", + # flake8-type-checking + "TC", ] ignore = [ # Assigning to `os.environ` doesn't clear the environment. diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 4f6933006..b96c63eb7 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import subprocess # nosec import sys import time @@ -13,6 +12,7 @@ from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: + import argparse from collections.abc import Iterable from scrapy import Request diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a1806f626..05e5e53e9 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING from w3lib.url import is_url -from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.http import Request, Response @@ -15,6 +14,8 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request if TYPE_CHECKING: from argparse import ArgumentParser, Namespace + from scrapy import Spider + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index b286e703e..2e70b2865 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,12 +1,11 @@ from __future__ import annotations -import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse import scrapy @@ -14,6 +13,9 @@ from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase +if TYPE_CHECKING: + import argparse + def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fba294851..fc16e46d1 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import functools import inspect import json @@ -22,6 +21,7 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: + import argparse from collections.abc import AsyncGenerator, Coroutine, Iterable from twisted.python.failure import Failure diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 7ec56899c..55211f8d7 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import sys from importlib import import_module from pathlib import Path @@ -11,6 +10,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + import argparse from os import PathLike from types import ModuleType diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 27e6d68ee..4ca015f5e 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,7 +9,6 @@ from __future__ import annotations from threading import Thread from typing import TYPE_CHECKING, Any -from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -19,6 +18,8 @@ from scrapy.utils.url import guess_scheme if TYPE_CHECKING: from argparse import ArgumentParser, Namespace + from scrapy import Spider + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index f54c02369..6da877610 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import os import re import string @@ -8,12 +7,16 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import TYPE_CHECKING import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase +if TYPE_CHECKING: + import argparse + TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5040741e2..434b316e9 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached @@ -23,6 +22,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings + from scrapy.signalmanager import SignalManager _T = TypeVar("_T") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 20377ac06..7f3da67eb 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from collections.abc import Callable from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet import defer @@ -15,7 +14,7 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Callable, Generator from twisted.internet.defer import Deferred diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 60cffae35..5480df72c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,13 +17,9 @@ from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals -from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper, _HandleOutputDeferred from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response -from scrapy.logformatter import LogFormatter -from scrapy.settings import Settings -from scrapy.signalmanager import SignalManager from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce @@ -31,9 +27,12 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Iterator + from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.logformatter import LogFormatter + from scrapy.settings import BaseSettings, Settings + from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 83dad0c0b..03301717d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,9 +16,6 @@ from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.logformatter import LogFormatter -from scrapy.pipelines import ItemPipelineManager -from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( aiter_errback, defer_fail, @@ -35,6 +32,9 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatter + from scrapy.pipelines import ItemPipelineManager + from scrapy.signalmanager import SignalManager logger = logging.getLogger(__name__) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1ad837a47..05af1bf8a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -18,10 +18,8 @@ from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader -from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -42,7 +40,9 @@ from scrapy.utils.reactor import ( if TYPE_CHECKING: from collections.abc import Generator, Iterable + from scrapy.logformatter import LogFormatter from scrapy.spiderloader import SpiderLoader + from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinter diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c9350a956..9380b7e78 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -10,8 +10,7 @@ import pickle # nosec import pprint from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper -from json import JSONEncoder -from typing import Any +from typing import TYPE_CHECKING, Any from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -21,6 +20,9 @@ from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + from json import JSONEncoder + __all__ = [ "BaseItemExporter", "PprintItemExporter", diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0cf44aed8..f6415ad8e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.settings import Settings from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.ftp import ftp_store_file @@ -44,7 +43,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.exporters import BaseItemExporter - from scrapy.settings import BaseSettings + from scrapy.settings import BaseSettings, Settings logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 0e6120c21..0edcce888 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -2,13 +2,11 @@ from __future__ import annotations import gzip import logging -import os import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time -from types import ModuleType from typing import IO, TYPE_CHECKING, Any, cast from weakref import WeakKeyDictionary @@ -19,10 +17,11 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: + import os from collections.abc import Callable + from types import ModuleType # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate @@ -30,6 +29,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider + from scrapy.utils.request import RequestFingerprinter logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index f2e3782a4..7cf08a1bb 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging from datetime import datetime, timezone -from json import JSONEncoder from typing import TYPE_CHECKING, Any from twisted.internet import task @@ -13,6 +12,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: # typing.Self requires Python 3.11 + from json import JSONEncoder + from typing_extensions import Self from scrapy.crawler import Crawler diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 07dc5880b..89c83d20d 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -13,7 +13,6 @@ import pprint from typing import TYPE_CHECKING, Any from twisted.internet import protocol -from twisted.internet.tcp import Port from scrapy import signals from scrapy.exceptions import NotConfigured @@ -24,6 +23,7 @@ from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: from twisted.conch import telnet + from twisted.internet.tcp import Port # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9c29ea4d1..107469504 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -21,7 +21,6 @@ from typing import ( from w3lib.url import safe_url_string -import scrapy from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes @@ -37,6 +36,7 @@ if TYPE_CHECKING: # typing.NotRequired and typing.Self require Python 3.11 from typing_extensions import Concatenate, NotRequired, Self + from scrapy import Spider from scrapy.http import Response CallbackT = Callable[Concatenate[Response, ...], Any] @@ -252,7 +252,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: + def to_dict(self, *, spider: Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 28e2073a2..4dea5afea 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -5,7 +5,6 @@ import logging from typing import TYPE_CHECKING, Protocol, cast from scrapy import Request -from scrapy.core.downloader import Downloader from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: @@ -14,6 +13,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler logger = logging.getLogger(__name__) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e621525f2..a86aad51c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,7 +3,6 @@ from __future__ import annotations import numbers import os import sys -from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path @@ -15,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values if TYPE_CHECKING: - from collections.abc import Collection, Mapping, MutableMapping + from collections.abc import Collection, Iterable, Mapping, MutableMapping def build_component_list( diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ac4358410..2102ce798 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error -from twisted.internet.base import DelayedCall from scrapy.utils.misc import load_object @@ -15,6 +14,7 @@ if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from collections.abc import Callable + from twisted.internet.base import DelayedCall from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 92b73a91a..a7b84baef 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -16,7 +16,6 @@ from unittest import TestCase, mock from twisted.trial.unittest import SkipTest from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available @@ -26,6 +25,8 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse + from scrapy.crawler import Crawler + _T = TypeVar("_T")