diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fbf74058..bcc10d5e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,8 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.3 + rev: v0.12.2 hooks: - - id: ruff + - id: ruff-check args: [ --fix ] - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs @@ -10,7 +10,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==24.10.0 + - black==25.1.0 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/conftest.py b/conftest.py index f952127b9..02697603d 100644 --- a/conftest.py +++ b/conftest.py @@ -85,7 +85,7 @@ def requires_uvloop(request): if not request.node.get_closest_marker("requires_uvloop"): return try: - import uvloop + import uvloop # noqa: PLC0415 del uvloop except ImportError: @@ -97,7 +97,7 @@ def requires_botocore(request): if not request.node.get_closest_marker("requires_botocore"): return try: - import botocore + import botocore # noqa: PLC0415 del botocore except ImportError: @@ -109,7 +109,7 @@ def requires_boto3(request): if not request.node.get_closest_marker("requires_boto3"): return try: - import boto3 + import boto3 # noqa: PLC0415 del boto3 except ImportError: diff --git a/pyproject.toml b/pyproject.toml index a6b99ac5b..b25ccb894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,10 +106,6 @@ follow_imports = "skip" module = "scrapy.settings.default_settings" ignore_errors = true -[[tool.mypy.overrides]] -module = "itemadapter" -implicit_reexport = true - [[tool.mypy.overrides]] module = "twisted" implicit_reexport = true @@ -394,6 +390,9 @@ banned-module-level-imports = [ "twisted.internet.reactor", ] +[tool.ruff.lint.isort] +split-on-trailing-comma = false + [tool.ruff.lint.per-file-ignores] # Circular import workarounds "scrapy/linkextractors/__init__.py" = ["E402"] diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 256504c9c..280a0f659 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -31,11 +31,11 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("." def __getattr__(name: str): if name == "twisted_version": - import warnings # pylint: disable=reimported + import warnings # noqa: PLC0415 # pylint: disable=reimported - from twisted import version as _txv + from twisted import version as _txv # noqa: PLC0415 - from scrapy.exceptions import ScrapyDeprecationWarning + from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 warnings.warn( "The scrapy.twisted_version attribute is deprecated, use twisted.version instead", diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 96bb1ae84..c4e277a60 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -10,6 +10,7 @@ import scrapy from scrapy.commands import ScrapyCommand from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor +from scrapy.utils.test import get_testenv if TYPE_CHECKING: import argparse @@ -35,8 +36,6 @@ class Command(ScrapyCommand): class _BenchServer: def __enter__(self) -> None: - from scrapy.utils.test import get_testenv - pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] self.proc = subprocess.Popen( # noqa: S603 pargs, stdout=subprocess.PIPE, env=get_testenv() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 0e90c3188..9d2742afd 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -206,7 +206,7 @@ class Command(ScrapyCommand): # a file with the same name exists in the target directory spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) - spiders_dir = Path(cast(str, spiders_module.__file__)).parent + spiders_dir = Path(cast("str", spiders_module.__file__)).parent spiders_dir_abs = spiders_dir.resolve() path = spiders_dir_abs / (name + ".py") if path.exists(): diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index bdb68c4ad..f5ba5ba13 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -52,7 +52,7 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - return list(cast(Iterable[Any], iterate_spider_output(cb_result))) + return list(cast("Iterable[Any]", iterate_spider_output(cb_result))) request.callback = wrapper @@ -68,7 +68,7 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - output = list(cast(Iterable[Any], iterate_spider_output(cb_result))) + output = list(cast("Iterable[Any]", iterate_spider_output(cb_result))) try: results.startTest(self.testcase_post) self.post_process(output) @@ -181,7 +181,7 @@ class ContractsManager: def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) - output = list(cast(Iterable[Any], iterate_spider_output(output))) + output = list(cast("Iterable[Any]", iterate_spider_output(output))) except Exception: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c0833c84d..4c9fed3bc 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -162,7 +162,7 @@ class Downloader: def get_slot_key(self, request: Request) -> str: if self.DOWNLOAD_SLOT in request.meta: - return cast(str, request.meta[self.DOWNLOAD_SLOT]) + return cast("str", request.meta[self.DOWNLOAD_SLOT]) key = urlparse_cached(request).hostname or "" if self.ip_concurrency: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 54fef48b6..50235846b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -386,6 +386,7 @@ class ScrapyAgent: if not proxy_port: proxy_port = 443 if proxy_parsed.scheme == "https" else 80 if urlparse_cached(request).scheme == "https": + assert proxy_host is not None proxyAuth = request.headers.get(b"Proxy-Authorization", None) proxyConf = (proxy_host, proxy_port, proxyAuth) return self._TunnelingAgent( @@ -430,7 +431,7 @@ class ScrapyAgent: method, to_bytes(url, encoding="ascii"), headers, - cast(IBodyProducer, bodyproducer), + cast("IBodyProducer", bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 870a26f04..02beb2f8b 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -51,8 +51,8 @@ class S3DownloadHandler: self.anon = kw.get("anon") self._signer = None - import botocore.auth - import botocore.credentials + import botocore.auth # noqa: PLC0415 + import botocore.credentials # noqa: PLC0415 kw.pop("anon", None) if kw: @@ -87,7 +87,7 @@ class S3DownloadHandler: if self.anon: request = request.replace(url=url) else: - import botocore.awsrequest + import botocore.awsrequest # noqa: PLC0415 awsrequest = botocore.awsrequest.AWSRequest( method=request.method, diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index fa45d62ee..2e892cff7 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,7 +6,6 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from collections.abc import Callable from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -18,7 +17,7 @@ from scrapy.utils.conf import build_component_list from scrapy.utils.defer import _defer_sleep, deferred_from_coro if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Callable, Generator from scrapy import Spider from scrapy.settings import BaseSettings @@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): request: Request, ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_request"]: - method = cast(Callable, method) + method = cast("Callable", method) response = yield deferred_from_coro( method(request=request, spider=spider) ) @@ -76,7 +75,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods["process_response"]: - method = cast(Callable, method) + method = cast("Callable", method) response = yield deferred_from_coro( method(request=request, response=response, spider=spider) ) @@ -94,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): exception: Exception, ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_exception"]: - method = cast(Callable, method) + method = cast("Callable", method) response = yield deferred_from_coro( method(request=request, exception=exception, spider=spider) ) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index be76ad8c3..970a250ef 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,13 +17,11 @@ from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, su from twisted.python.failure import Failure from scrapy import signals +from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.asyncio import ( - AsyncioLoopingCall, - create_looping_call, -) +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import ( deferred_f_from_coro_f, deferred_from_coro, @@ -39,7 +37,6 @@ if TYPE_CHECKING: from twisted.internet.task import LoopingCall from scrapy.core.downloader import Downloader - from scrapy.core.scheduler import BaseScheduler from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -123,8 +120,6 @@ class ExecutionEngine: raise def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: - from scrapy.core.scheduler import BaseScheduler - scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( @@ -506,7 +501,7 @@ class ExecutionEngine: dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self._slot.scheduler, "close"): - dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason)) + dfd.addBoth(lambda _: cast("_Slot", self._slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 9ac447289..9ae555300 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -491,7 +491,7 @@ class Scheduler(BaseScheduler): if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(list[int], json.load(f)) + return cast("list[int]", json.load(f)) def _write_dqs_state(self, dqdir: str, state: list[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 77fcaa357..01e563e56 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -138,7 +138,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred[Iterable[_T] | AsyncIterator[_T]]: for method in self.methods["process_spider_input"]: - method = cast(Callable, method) + method = cast("Callable", method) try: result = method(response=response, spider=spider) if result is not None: @@ -166,7 +166,7 @@ class SpiderMiddlewareManager(MiddlewareManager): yield from iterable except Exception as ex: exception_result = cast( - Union[Failure, MutableChain[_T]], + "Union[Failure, MutableChain[_T]]", self._process_spider_exception( response, spider, ex, exception_processor_index ), @@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager): yield r except Exception as ex: exception_result = cast( - Union[Failure, MutableAsyncChain[_T]], + "Union[Failure, MutableAsyncChain[_T]]", self._process_spider_exception( response, spider, ex, exception_processor_index ), @@ -212,7 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager): for method_index, method in enumerate(method_list, start=start_index): if method is None: continue - method = cast(Callable, method) + method = cast("Callable", method) result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the @@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading return cast( - Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result + "Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result ) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d6fb9972e..c6c65a993 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -8,11 +8,7 @@ import signal from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, TypeVar -from twisted.internet.defer import ( - Deferred, - DeferredList, - inlineCallbacks, -) +from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from scrapy import Spider, signals from scrapy.addons import AddonManager diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 612426371..530cccb53 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -202,6 +202,6 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): redirected = self._redirect_request_using_get(request, url) if urlparse_cached(redirected).scheme not in {"http", "https"}: return response - if cast(float, interval) < self._maxdelay: + if cast("float", interval) < self._maxdelay: return self._redirect(redirected, request, spider, "meta refresh") return response diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 557ad99e1..0847c1e8b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -218,7 +218,7 @@ class S3FeedStorage(BlockingFeedStorage): region_name: str | None = None, ): try: - import boto3.session + import boto3.session # noqa: PLC0415 except ImportError: raise NotConfigured("missing boto3 library") u = urlparse(uri) @@ -317,7 +317,7 @@ class GCSFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa: PLC0415 client = Client(project=self.project_id) bucket = client.get_bucket(self.bucket_name) @@ -413,7 +413,7 @@ class FeedSlot: self.file = self.storage.open(self.spider) if "postprocessing" in self.feed_options: self.file = cast( - IO[bytes], + "IO[bytes]", PostProcessingManager( self.feed_options["postprocessing"], self.file, @@ -662,7 +662,7 @@ class FeedExporter: def _load_components(self, setting_prefix: str) -> dict[str, Any]: conf = without_none_values( - cast(dict[str, str], self.settings.getwithbase(setting_prefix)) + cast("dict[str, str]", self.settings.getwithbase(setting_prefix)) ) d = {} for k, v in conf.items(): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 0cd16d737..f704ae014 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -307,7 +307,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return None # expired - return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301 + return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"])) # noqa: S301 class FilesystemCacheStorage: @@ -389,7 +389,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return None # expired with self._open(metapath, "rb") as f: - return cast(dict[str, Any], pickle.load(f)) # noqa: S301 + return cast("dict[str, Any]", pickle.load(f)) # noqa: S301 def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]: diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 387cfddb3..3d7674905 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -5,10 +5,7 @@ from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured -from scrapy.utils.asyncio import ( - AsyncioLoopingCall, - create_looping_call, -) +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call if TYPE_CHECKING: from twisted.internet.task import LoopingCall diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 2ef322f1a..e425749f7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -16,10 +16,7 @@ from typing import TYPE_CHECKING from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.utils.asyncio import ( - AsyncioLoopingCall, - create_looping_call, -) +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.engine import get_engine_status if TYPE_CHECKING: diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 98210990a..860b97a55 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Spider, signals from scrapy.exceptions import NotConfigured -from scrapy.utils.asyncio import ( - AsyncioLoopingCall, - create_looping_call, -) +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index b1fa160c8..01fd82830 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -142,7 +142,7 @@ class PostProcessingManager(IOBase): :return: returns number of bytes written :rtype: int """ - return cast(int, self.head_plugin.write(data)) + return cast("int", self.head_plugin.write(data)) def tell(self) -> int: return self.file.tell() diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index bacee8f0a..9fe71817a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -12,6 +12,8 @@ import os import pprint from typing import TYPE_CHECKING, Any +from twisted.conch import manhole, telnet +from twisted.conch.insults import insults from twisted.internet import protocol from scrapy import signals @@ -22,7 +24,6 @@ from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: - from twisted.conch import telnet from twisted.internet.tcp import Port # typing.Self requires Python 3.11 @@ -76,10 +77,6 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self) -> telnet.TelnetTransport: - # these import twisted.internet.reactor - from twisted.conch import manhole, telnet - from twisted.conch.insults import insults - class Portal: """An implementation of IPortal""" diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index b7c3b9d37..e5b4b28a7 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -161,7 +161,7 @@ class WrappedRequest: HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. """ - return cast(bool, self.request.meta.get("is_unverifiable", False)) + return cast("bool", self.request.meta.get("is_unverifiable", False)) @property def full_url(self) -> str: @@ -181,7 +181,7 @@ class WrappedRequest: @property def origin_req_host(self) -> str: - return cast(str, urlparse_cached(self.request).hostname) + return cast("str", urlparse_cached(self.request).hostname) def has_header(self, name: str) -> bool: return name in self.request.headers diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 60b04753b..5498e1138 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -69,19 +69,19 @@ class Headers(CaselessDict): def __getitem__(self, key: AnyStr) -> bytes | None: try: - return cast(list[bytes], super().__getitem__(key))[-1] + return cast("list[bytes]", super().__getitem__(key))[-1] except IndexError: return None def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: try: - return cast(list[bytes], super().get(key, def_val))[-1] + return cast("list[bytes]", super().get(key, def_val))[-1] except IndexError: return None def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: try: - return cast(list[bytes], super().__getitem__(key)) + return cast("list[bytes]", super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7681419c4..aa15a0222 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -11,13 +11,7 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import ( - FormElement, - InputElement, - MultipleSelectOptions, - SelectElement, - TextareaElement, -) +from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request @@ -25,6 +19,13 @@ from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 + from lxml.html import ( + FormElement, + InputElement, + MultipleSelectOptions, + SelectElement, + TextareaElement, + ) from typing_extensions import Self from scrapy.http.response.text import TextResponse @@ -76,8 +77,6 @@ class FormRequest(Request): kwargs.setdefault("encoding", response.encoding) if formcss is not None: - from parsel.csstranslator import HTMLTranslator - formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) @@ -107,7 +106,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: values = [ (to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq - for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)]) + for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)]) ] return urlencode(values, doseq=True) @@ -128,12 +127,12 @@ def _get_form( if formname is not None: f = root.xpath(f'//form[@name="{formname}"]') if f: - return cast(FormElement, f[0]) + return cast("FormElement", f[0]) if formid is not None: f = root.xpath(f'//form[@id="{formid}"]') if f: - return cast(FormElement, f[0]) + return cast("FormElement", f[0]) # Get form element from xpath, if not found, go up if formxpath is not None: @@ -142,7 +141,7 @@ def _get_form( el = nodes[0] while True: if el.tag == "form": - return cast(FormElement, el) + return cast("FormElement", el) el = el.getparent() if el is None: break @@ -153,7 +152,7 @@ def _get_form( form = forms[formnumber] except IndexError: raise IndexError(f"Form number {formnumber} not found in {response}") - return cast(FormElement, form) + return cast("FormElement", form) def _get_inputs( @@ -201,7 +200,7 @@ def _value( n = ele.name v = ele.value if ele.tag == "select": - return _select_value(cast(SelectElement, ele), n, v) + return _select_value(cast("SelectElement", ele), n, v) return n, v @@ -251,7 +250,7 @@ def _get_clickable( except IndexError: pass else: - return (el.get("name"), el.get("value") or "") + return (cast("str", el.get("name")), el.get("value") or "") # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 081223882..9c4e4c151 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -104,13 +104,14 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self) -> str | None: - content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) + content_type = cast("bytes", self.headers.get(b"Content-Type", b"")) return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self) -> str: if self._cached_benc is None: content_type = to_unicode( - cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1" + cast("bytes", self.headers.get(b"Content-Type", b"")), + encoding="latin-1", ) benc, ubody = html_to_unicode( content_type, @@ -141,31 +142,25 @@ class TextResponse(Response): @property def selector(self) -> Selector: - from scrapy.selector import Selector + # circular import + from scrapy.selector import Selector # noqa: PLC0415 if self._cached_selector is None: self._cached_selector = Selector(self) return self._cached_selector def jmespath(self, query: str, **kwargs: Any) -> SelectorList: - from scrapy.selector import SelectorList - if not hasattr(self.selector, "jmespath"): raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - - return cast(SelectorList, self.selector.jmespath(query, **kwargs)) + return cast("SelectorList", self.selector.jmespath(query, **kwargs)) def xpath(self, query: str, **kwargs: Any) -> SelectorList: - from scrapy.selector import SelectorList - - return cast(SelectorList, self.selector.xpath(query, **kwargs)) + return cast("SelectorList", self.selector.xpath(query, **kwargs)) def css(self, query: str) -> SelectorList: - from scrapy.selector import SelectorList - - return cast(SelectorList, self.selector.css(query)) + return cast("SelectorList", self.selector.css(query)) def follow( self, diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 814e31fec..7c9b4d3e3 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -71,12 +71,12 @@ class LxmlParserLinkExtractor: self.scan_tag: Callable[[str], bool] = ( tag if callable(tag) - else cast(Callable[[str], bool], partial(operator.eq, tag)) + else cast("Callable[[str], bool]", partial(operator.eq, tag)) ) self.scan_attr: Callable[[str], bool] = ( attr if callable(attr) - else cast(Callable[[str], bool], partial(operator.eq, attr)) + else cast("Callable[[str], bool]", partial(operator.eq, attr)) ) self.process_attr: Callable[[Any], Any] = ( process if callable(process) else _identity @@ -84,7 +84,7 @@ class LxmlParserLinkExtractor: self.unique: bool = unique self.strip: bool = strip self.link_key: Callable[[Link], str] = ( - cast(Callable[[Link], str], operator.attrgetter("url")) + cast("Callable[[Link], str]", operator.attrgetter("url")) if canonicalized else _canonicalize_link_url ) diff --git a/scrapy/mail.py b/scrapy/mail.py index be2423965..88612daa9 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -223,7 +223,8 @@ class MailSender: def _create_sender_factory( self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: - from twisted.mail.smtp import ESMTPSenderFactory + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory # noqa: PLC0415 factory_keywords: dict[str, Any] = { "heloFallback": True, diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 25f2aa199..a4d7af7b6 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -63,7 +63,7 @@ class MiddlewareManager(ABC): method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return cast(_T, instance) + return cast("_T", instance) @classmethod def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 55d9cae6c..84d4104ed 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -169,7 +169,7 @@ class S3FilesStore: def __init__(self, uri: str): if not is_botocore_available(): raise NotConfigured("missing botocore library") - import botocore.session + import botocore.session # noqa: PLC0415 session = botocore.session.get_session() self.s3_client = session.create_client( @@ -284,7 +284,7 @@ class GCSFilesStore: POLICY = None def __init__(self, uri: str): - from google.cloud import storage + from google.cloud import storage # noqa: PLC0415 client = storage.Client(project=self.GCS_PROJECT_ID) bucket, prefix = uri[5:].split("/", 1) @@ -317,7 +317,7 @@ class GCSFilesStore: blob_path = self._get_blob_path(path) return cast( - Deferred[StatInfo], + "Deferred[StatInfo]", deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) @@ -531,7 +531,9 @@ class FilesPipeline(MediaPipeline): @classmethod def _update_stores(cls, settings: BaseSettings) -> None: - s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast( + "type[S3FilesStore]", cls.STORE_SCHEMES["s3"] + ) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -542,13 +544,13 @@ class FilesPipeline(MediaPipeline): s3store.POLICY = settings["FILES_STORE_S3_ACL"] gcs_store: type[GCSFilesStore] = cast( - type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None ftp_store: type[FTPFilesStore] = cast( - type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + "type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -742,5 +744,5 @@ class FilesPipeline(MediaPipeline): media_ext = "" media_type = mimetypes.guess_type(request.url)[0] if media_type: - media_ext = cast(str, mimetypes.guess_extension(media_type)) + media_ext = cast("str", mimetypes.guess_extension(media_type)) return f"full/{media_guid}{media_ext}" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 63c6908dc..19139b5d5 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -63,7 +63,7 @@ class ImagesPipeline(FilesPipeline): crawler: Crawler | None = None, ): try: - from PIL import Image + from PIL import Image # noqa: PLC0415 self._Image = Image except ImportError: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 34b235d83..42c53a527 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -4,7 +4,6 @@ import hashlib import logging from typing import TYPE_CHECKING, Protocol, cast -from scrapy import Request from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: @@ -13,6 +12,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler @@ -211,7 +211,7 @@ class ScrapyPriorityQueue: except KeyError: queue = self.queues[self.curprio] # Protocols can't declare optional members - return cast(Request, queue.peek()) # type: ignore[attr-defined] + return cast("Request", queue.peek()) # type: ignore[attr-defined] def close(self) -> list[int]: active: set[int] = set() diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 417c9c142..e1a12be05 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,6 +4,9 @@ import logging import sys from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING +from urllib.robotparser import RobotFileParser + +from protego import Protego from scrapy.utils.python import to_unicode @@ -67,8 +70,6 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): - from urllib.robotparser import RobotFileParser - self.spider: Spider | None = spider body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp: RobotFileParser = RobotFileParser() @@ -87,7 +88,7 @@ class PythonRobotParser(RobotParser): class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): - from robotexclusionrulesparser import RobotExclusionRulesParser + from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: PLC0415 self.spider: Spider | None = spider self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() @@ -107,8 +108,6 @@ class RerpRobotParser(RobotParser): class ProtegoRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): - from protego import Protego - self.spider: Spider | None = spider body_decoded = decode_robotstxt(robotstxt_body, spider) self.rp = Protego.parse(body_decoded) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 9363da492..334de6658 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -336,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): stored. """ if len(self) > 0: - return max(cast(int, self.getpriority(name)) for name in self) + return max(cast("int", self.getpriority(name)) for name in self) return get_settings_priority("default") def replace_in_component_priority_dict( @@ -519,11 +519,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(dict[_SettingsKeyT, Any], json.loads(values)) + values = cast("dict[_SettingsKeyT, Any]", json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): - self.set(name, value, cast(int, values.getpriority(name))) + self.set(name, value, cast("int", values.getpriority(name))) else: for name, value in values.items(): self.set(name, value, priority) @@ -533,7 +533,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): raise KeyError(name) self._assert_mutability() priority = get_settings_priority(priority) - if priority >= cast(int, self.getpriority(name)): + if priority >= cast("int", self.getpriority(name)): del self.attributes[name] def __delitem__(self, name: _SettingsKeyT) -> None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 50c0088e7..f306569e4 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -535,9 +535,9 @@ WARN_ON_GENERATOR_RETURN_VALUE = True def __getattr__(name: str): if name == "CONCURRENT_REQUESTS_PER_IP": - import warnings + import warnings # noqa: PLC0415 - from scrapy.exceptions import ScrapyDeprecationWarning + from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 warnings.warn( "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", diff --git a/scrapy/shell.py b/scrapy/shell.py index bb39eccc3..c3a274e0d 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -16,6 +16,7 @@ from twisted.internet import defer, threads from twisted.python import threadable from w3lib.url import any_to_uri +import scrapy from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response @@ -164,8 +165,6 @@ class Shell: request: Request | None = None, spider: Spider | None = None, ) -> None: - import scrapy - self.vars["scrapy"] = scrapy self.vars["crawler"] = self.crawler self.vars["item"] = self.item_class() diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1d4d1527b..e4d0f4014 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -315,7 +315,7 @@ def _load_policy_class( from https://www.w3.org/TR/referrer-policy/#referrer-policies """ try: - return cast(type[ReferrerPolicy], load_object(policy)) + return cast("type[ReferrerPolicy]", load_object(policy)) except ValueError: tokens = [token.strip() for token in policy.lower().split(",")] # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index a722dd83b..8db00a1ca 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -55,7 +55,8 @@ class Spider(object_ref): @property def logger(self) -> SpiderLoggerAdapter: - from scrapy.utils.log import SpiderLoggerAdapter + # circular import + from scrapy.utils.log import SpiderLoggerAdapter # noqa: PLC0415 logger = logging.getLogger(self.name) return SpiderLoggerAdapter(logger, {"spider": self}) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 98e7b23c0..0cbd6b3cc 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -12,8 +12,6 @@ import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast -from twisted.python.failure import Failure - from scrapy.http import HtmlResponse, Request, Response from scrapy.link import Link from scrapy.linkextractors import LinkExtractor @@ -26,6 +24,8 @@ from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: from collections.abc import Iterable, Sequence + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -81,12 +81,14 @@ class Rule: def _compile(self, spider: Spider) -> None: # this replaces method names with methods and we can't express this in type hints self.callback = cast("CallbackT", _get_method(self.callback, spider)) - self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) + self.errback = cast( + "Callable[[Failure], Any]", _get_method(self.errback, spider) + ) self.process_links = cast( - ProcessLinksT, _get_method(self.process_links, spider) + "ProcessLinksT", _get_method(self.process_links, spider) ) self.process_request = cast( - ProcessRequestT, _get_method(self.process_request, spider) + "ProcessRequestT", _get_method(self.process_request, spider) ) @@ -141,13 +143,13 @@ class CrawlSpider(Spider): for lnk in rule.link_extractor.extract_links(response) if lnk not in seen ] - for link in cast(ProcessLinksT, rule.process_links)(links): + for link in cast("ProcessLinksT", rule.process_links)(links): seen.add(link) request = self._build_request(rule_index, link) - yield cast(ProcessRequestT, rule.process_request)(request, response) + yield cast("ProcessRequestT", rule.process_request)(request, response) def _callback(self, response: Response, **cb_kwargs: Any) -> Any: - rule = self._rules[cast(int, response.meta["rule"])] + rule = self._rules[cast("int", response.meta["rule"])] return self.parse_with_rules( response, cast("CallbackT", rule.callback), @@ -156,9 +158,9 @@ class CrawlSpider(Spider): ) def _errback(self, failure: Failure) -> Iterable[Any]: - rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined] + rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined] return self._handle_failure( - failure, cast(Callable[[Failure], Any], rule.errback) + failure, cast("Callable[[Failure], Any]", rule.errback) ) async def parse_with_rules( diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index e5548b9fa..957bfffd3 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,15 +1,16 @@ from __future__ import annotations import warnings -from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast -from scrapy import Request from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + from scrapy import Request from scrapy.http import Response @@ -39,7 +40,7 @@ class InitSpider(Spider): def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() - return cast(Iterable[Request], iterate_spider_output(self.init_request())) + return cast("Iterable[Request]", iterate_spider_output(self.init_request())) def initialized(self, response: Response | None = None) -> Any: """This method must be set as the callback of your last initialization diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 53cfeddd0..73f86bc71 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -3,7 +3,7 @@ def is_botocore_available() -> bool: try: - import botocore # noqa: F401 + import botocore # noqa: F401,PLC0415 return True except ImportError: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 891cbb485..5a627fc83 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -153,7 +153,7 @@ def feed_process_params_from_cli( suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS")) + cast("dict[str, str]", settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 7425543ff..644965cb5 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,5 +1,6 @@ from __future__ import annotations +import code from collections.abc import Callable from functools import wraps from typing import TYPE_CHECKING, Any @@ -16,13 +17,13 @@ def _embed_ipython_shell( ) -> EmbedFuncT: """Start an IPython Shell""" try: - from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100 - from IPython.terminal.ipapp import load_default_config + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100 + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100,PLC0415 InteractiveShellEmbed, ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] + from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] # noqa: PLC0415 load_default_config, ) @@ -46,7 +47,7 @@ def _embed_bpython_shell( namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a bpython shell""" - import bpython + import bpython # noqa: PLC0415 @wraps(_embed_bpython_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: @@ -59,7 +60,7 @@ def _embed_ptpython_shell( namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" - import ptpython.repl # pylint: disable=import-error + import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error @wraps(_embed_ptpython_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: @@ -73,14 +74,12 @@ def _embed_standard_shell( namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a standard python shell""" - import code - try: # readline module is only available on unix systems - import readline + import readline # noqa: PLC0415 except ImportError: pass else: - import rlcompleter # noqa: F401 + import rlcompleter # noqa: F401,PLC0415 readline.parse_and_bind("tab:complete") # type: ignore[attr-defined] diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 3d0e0d3c7..ae7f627e1 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -32,7 +32,8 @@ class CaselessDict(dict): __slots__ = () def __new__(cls, *args: Any, **kwargs: Any) -> Self: - from scrapy.http.headers import Headers + # circular import + from scrapy.http.headers import Headers # noqa: PLC0415 if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): warnings.warn( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 50f2af5b7..d161f30a6 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -12,12 +12,7 @@ from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload -from twisted.internet.defer import ( - Deferred, - DeferredList, - fail, - succeed, -) +from twisted.internet.defer import Deferred, DeferredList, fail, succeed from twisted.internet.task import Cooperator from twisted.python import failure @@ -414,7 +409,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: if not is_asyncio_available(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) + return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor return Deferred.fromFuture(asyncio.ensure_future(o)) return o diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 20744a604..08e89dc0d 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -34,11 +34,11 @@ def _colorize(text: str, colorize: bool = True) -> str: if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: - from pygments import highlight + from pygments import highlight # noqa: PLC0415 except ImportError: return text - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer + from pygments.formatters import TerminalFormatter # noqa: PLC0415 + from pygments.lexers import PythonLexer # noqa: PLC0415 return highlight(text, PythonLexer(), TerminalFormatter()) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c646fc218..c70a0d327 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -147,12 +147,12 @@ class _StreamReader: def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return cast(bytes, self._text)[s:e] + return cast("bytes", self._text)[s:e] def _read_unicode(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return cast(str, self._text)[s:e].encode("utf-8") + return cast("str", self._text)[s:e].encode("utf-8") def csviter( diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 24e17ecb6..533906003 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -5,9 +5,9 @@ import pprint import sys from collections.abc import MutableMapping from logging.config import dictConfig -from types import TracebackType from typing import TYPE_CHECKING, Any, Optional, cast +from twisted.internet import asyncioreactor from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -16,6 +16,8 @@ from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import get_versions if TYPE_CHECKING: + from types import TracebackType + from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatterResult @@ -33,7 +35,7 @@ def failure_to_exc_info( return ( failure.type, failure.value, - cast(Optional[TracebackType], failure.getTracebackObject()), + cast("Optional[TracebackType]", failure.getTracebackObject()), ) return None @@ -182,7 +184,7 @@ def log_scrapy_info(settings: Settings) -> None: def log_reactor_info() -> None: - from twisted.internet import asyncioreactor, reactor + from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): @@ -240,7 +242,7 @@ def logformatter_adapter( message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] + args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b7b436260..decd4473c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,7 +9,6 @@ import os import re import warnings from collections import deque -from collections.abc import Iterable from contextlib import contextmanager from functools import partial from importlib import import_module @@ -21,7 +20,7 @@ from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: - from collections.abc import Callable, Iterator + from collections.abc import Callable, Iterable, Iterator from types import ModuleType from scrapy import Spider @@ -41,7 +40,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: if arg is None: return [] if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return cast(Iterable[Any], arg) + return cast("Iterable[Any]", arg) return [arg] @@ -202,7 +201,7 @@ def build_from_crawler( method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return cast(T, instance) + return cast("T", instance) @contextmanager diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ce4c01731..373ad652c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503 """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -39,7 +39,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: return reactor.listenTCP(0, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) - for x in range(portrange[0], portrange[1] + 1): # noqa: RET503 + for x in range(portrange[0], portrange[1] + 1): try: return reactor.listenTCP(x, factory, interface=host) except error.CannotListenError: @@ -60,7 +60,8 @@ class CallLaterOnce(Generic[_T]): self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: - from scrapy.utils.asyncio import call_later + # circular import + from scrapy.utils.asyncio import call_later # noqa: PLC0415 if self._call is None: self._call = call_later(delay, self) @@ -70,7 +71,8 @@ class CallLaterOnce(Generic[_T]): self._call.cancel() def __call__(self) -> _T: - from scrapy.utils.asyncio import call_later + # circular import + from scrapy.utils.asyncio import call_later # noqa: PLC0415 self._call = None result = self._func(*self._a, **self._kw) @@ -82,7 +84,8 @@ class CallLaterOnce(Generic[_T]): return result async def wait(self): - from scrapy.utils.defer import maybe_deferred_to_future + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 d = Deferred() self._deferreds.append(d) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 76a6b7de6..b6550432c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -91,7 +91,8 @@ def open_in_browser( if "item name" not in response.body: open_in_browser(response) """ - from scrapy.http import HtmlResponse, TextResponse + # circular imports + from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415 # XXX: this implementation is a bit dirty and could be improved body = response.body diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index e0d9f4595..c0100ea62 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -24,7 +24,7 @@ class Sitemap: xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320 + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) rt = self._root.tag assert isinstance(rt, str) self.type = rt.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 3780ad23e..4c0764396 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,6 +7,7 @@ from __future__ import annotations import asyncio import os import warnings +from ftplib import FTP from importlib import import_module from pathlib import Path from posixpath import split @@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, cast from unittest import TestCase, mock from twisted.trial.unittest import SkipTest +from twisted.web.client import Agent +from scrapy.crawler import CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class @@ -59,7 +62,7 @@ def skip_if_no_boto() -> None: def get_gcs_content_and_delete( bucket: Any, path: str ) -> tuple[bytes, list[dict[str, str]], Any]: - from google.cloud import storage + from google.cloud import storage # noqa: PLC0415 warnings.warn( "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", @@ -83,8 +86,6 @@ def get_ftp_content_and_delete( password: str, use_active_mode: bool = False, ) -> bytes: - from ftplib import FTP - warnings.warn( "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", category=ScrapyDeprecationWarning, @@ -137,8 +138,6 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - from scrapy.crawler import CrawlerRunner - # When needed, useful settings can be added here, e.g. ones that prevent # deprecation warnings. settings: dict[str, Any] = { @@ -192,7 +191,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ - from google.cloud.storage import Blob, Bucket, Client + from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 warnings.warn( "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.", @@ -213,7 +212,6 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.internet import reactor - from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 10f764ab8..ce26b7f92 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -6,7 +6,6 @@ import warnings from typing import TYPE_CHECKING, cast from twisted.internet.defer import Deferred -from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol from scrapy.exceptions import ScrapyDeprecationWarning @@ -14,6 +13,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: from collections.abc import Iterable + from twisted.internet.error import ProcessTerminated from twisted.python.failure import Failure @@ -55,7 +55,7 @@ class ProcessTest: msg += "\n" msg += f"\n>>> stderr <<<\n{pp.err.decode()}" raise RuntimeError(msg) - return cast(int, pp.exitcode), pp.out, pp.err + return cast("int", pp.exitcode), pp.out, pp.err class TestProcessProtocol(ProcessProtocol): @@ -72,5 +72,5 @@ class TestProcessProtocol(ProcessProtocol): self.err += data def processEnded(self, status: Failure) -> None: - self.exitcode = cast(ProcessTerminated, status.value).exitCode + self.exitcode = cast("ProcessTerminated", status.value).exitCode self.deferred.callback(self) diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py index 68239e651..f6fead718 100644 --- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -21,7 +21,7 @@ def log_task_exception(task: Task) -> None: try: task.result() except Exception: - logging.exception("Crawl task failed") + logging.exception("Crawl task failed") # noqa: LOG015 process = AsyncCrawlerProcess() diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index 18ec81fed..c8c73ba15 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -3,6 +3,8 @@ from __future__ import annotations import os from pathlib import Path +import pytest + from tests.test_commands import TestCommandBase, TestProjectBase @@ -15,7 +17,16 @@ class TestGenspiderCommand(TestCommandBase): assert self.call("genspider", "test_name", "test.com") == 0 assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() - def test_template(self, tplname="crawl"): + @pytest.mark.parametrize( + "tplname", + [ + "basic", + "crawl", + "xmlfeed", + "csvfeed", + ], + ) + def test_template(self, tplname: str) -> None: args = [f"--template={tplname}"] if tplname else [] spname = "test_spider" spmodule = f"{self.project_name}.spiders.{spname}" @@ -35,15 +46,6 @@ class TestGenspiderCommand(TestCommandBase): ) assert modify_time_after == modify_time_before - def test_template_basic(self): - self.test_template("basic") - - def test_template_csvfeed(self): - self.test_template("csvfeed") - - def test_template_xmlfeed(self): - self.test_template("xmlfeed") - def test_list(self): assert self.call("genspider", "--list") == 0 @@ -57,7 +59,8 @@ class TestGenspiderCommand(TestCommandBase): self.proj_mod_path, "spiders", f"{self.project_name}.py" ).exists() - def test_same_filename_as_existing_spider(self, force=False): + @pytest.mark.parametrize("force", [True, False]) + def test_same_filename_as_existing_spider(self, force: bool) -> None: file_name = "example" file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") assert self.call("genspider", file_name, "example.com") == 0 @@ -90,83 +93,60 @@ class TestGenspiderCommand(TestCommandBase): file_contents_after = file_path.read_text(encoding="utf-8") assert file_contents_after == file_contents_before - def test_same_filename_as_existing_spider_force(self): - self.test_same_filename_as_existing_spider(force=True) - - def test_url(self, url="test.com", domain="test.com"): + @pytest.mark.parametrize( + ("url", "domain"), + [ + ("test.com", "test.com"), + ("https://test.com", "test.com"), + ], + ) + def test_url(self, url: str, domain: str) -> None: assert self.call("genspider", "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == domain + m = self.find_in_file( + self.proj_mod_path / "spiders" / "test_name.py", + r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", ) - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == f"https://{domain}" + assert m is not None + assert m.group(1) == domain + m = self.find_in_file( + self.proj_mod_path / "spiders" / "test_name.py", + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", ) + assert m is not None + assert m.group(1) == f"https://{domain}" - def test_url_schema(self): - self.test_url("https://test.com", "test.com") - - def test_template_start_urls( - self, url="test.com", expected="https://test.com", template="basic" - ): + @pytest.mark.parametrize( + ("url", "expected", "template"), + [ + # basic + ("https://test.com", "https://test.com", "basic"), + ("http://test.com", "http://test.com", "basic"), + ("http://test.com/other/path", "http://test.com/other/path", "basic"), + ("test.com/other/path", "https://test.com/other/path", "basic"), + # crawl + ("https://test.com", "https://test.com", "crawl"), + ("http://test.com", "http://test.com", "crawl"), + ("http://test.com/other/path", "http://test.com/other/path", "crawl"), + ("test.com/other/path", "https://test.com/other/path", "crawl"), + ("test.com", "https://test.com", "crawl"), + # xmlfeed + ("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"), + ("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"), + ("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"), + # csvfeed + ("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"), + ("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"), + ("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"), + ], + ) + def test_template_start_urls(self, url: str, expected: str, template: str) -> None: assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == expected - ) - - def test_genspider_basic_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "basic") - self.test_template_start_urls("http://test.com", "http://test.com", "basic") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "basic" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "basic" - ) - - def test_genspider_crawl_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "crawl") - self.test_template_start_urls("http://test.com", "http://test.com", "crawl") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "crawl" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "crawl" - ) - self.test_template_start_urls("test.com", "https://test.com", "crawl") - - def test_genspider_xmlfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - - def test_genspider_csvfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" - ) - self.test_template_start_urls( - "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + m = self.find_in_file( + self.proj_mod_path / "spiders" / "test_name.py", + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", ) + assert m is not None + assert m.group(1) == expected class TestGenspiderStandaloneCommand(TestProjectBase): @@ -174,7 +154,8 @@ class TestGenspiderStandaloneCommand(TestProjectBase): self.call("genspider", "example", "example.com") assert Path(self.temp_path, "example.py").exists() - def test_same_name_as_existing_file(self, force=False): + @pytest.mark.parametrize("force", [True, False]) + def test_same_name_as_existing_file(self, force: bool) -> None: file_name = "example" file_path = Path(self.temp_path, file_name + ".py") p, out, err = self.proc("genspider", file_name, "example.com") @@ -203,6 +184,3 @@ class TestGenspiderStandaloneCommand(TestProjectBase): assert modify_time_after == modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") assert file_contents_after == file_contents_before - - def test_same_name_as_existing_file_force(self): - self.test_same_name_as_existing_file(force=True) diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 0c12a59f7..89670feb3 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import inspect import platform import sys @@ -204,8 +205,6 @@ class MySpider(scrapy.Spider): "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", ], ) - import asyncio - if sys.platform != "win32": loop = asyncio.new_event_loop() else: diff --git a/tests/test_commands.py b/tests/test_commands.py index a749de9fe..9ea893f05 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -9,7 +9,7 @@ from io import StringIO from pathlib import Path from shutil import rmtree from tempfile import TemporaryFile, mkdtemp -from typing import TYPE_CHECKING, Any +from typing import Any from unittest import mock import pytest @@ -22,9 +22,6 @@ from scrapy.utils.python import to_unicode from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_testenv -if TYPE_CHECKING: - import os - class EmptyCommand(ScrapyCommand): def short_desc(self) -> str: @@ -110,10 +107,11 @@ class TestProjectBase: return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None: + @staticmethod + def find_in_file(filename: Path, regex: str) -> re.Match | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) - with Path(filename).open("r", encoding="utf-8") as f: + with filename.open("r", encoding="utf-8") as f: for line in f: match = pattern.search(line) if match is not None: diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 464320602..0a674d638 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,7 @@ import shutil import warnings from pathlib import Path from tempfile import mkdtemp -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import OpenSSL.SSL import pytest @@ -14,7 +14,6 @@ from twisted.trial import unittest from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from twisted.web.iweb import IBodyProducer from scrapy.core.downloader import Slot from scrapy.core.downloader.contextfactory import ( @@ -29,6 +28,9 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler from tests.mockserver import PayloadResource, ssl_context_factory +if TYPE_CHECKING: + from twisted.web.iweb import IBodyProducer + class TestSlot: def test_repr(self): @@ -78,12 +80,12 @@ class TestContextFactoryBase(unittest.TestCase): agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None response: TxResponse = cast( - TxResponse, + "TxResponse", await maybe_deferred_to_future( agent.request( b"GET", url.encode(), - bodyProducer=cast(IBodyProducer, body_producer), + bodyProducer=cast("IBodyProducer", body_producer), ) ), ) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 42f9899f9..ef998ab4c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,7 +5,7 @@ import logging from ipaddress import IPv4Address from socket import gethostbyname from typing import TYPE_CHECKING, Any -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse import pytest from testfixtures import LogCapture @@ -20,6 +20,7 @@ from scrapy.exceptions import CloseSpider, StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE @@ -255,8 +256,6 @@ class TestCrawl(TestCase): def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" - from urllib.parse import urlencode - query = urlencode( { "raw": """\ @@ -339,8 +338,6 @@ with multiples lines @inlineCallbacks def test_engine_status(self): - from scrapy.utils.engine import get_engine_status - est = [] def cb(response): @@ -357,8 +354,6 @@ with multiples lines @inlineCallbacks def test_format_engine_status(self): - from scrapy.utils.engine import format_engine_status - est = [] def cb(response): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 56cb21650..e44bf1a46 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -526,10 +526,10 @@ class TestCrawlerLogging: crawler = get_crawler(MySpider) assert get_scrapy_root_handler().level == logging.INFO info_count = crawler.stats.get_value("log_count/INFO") - logging.debug("debug message") - logging.info("info message") - logging.warning("warning message") - logging.error("error message") + logging.debug("debug message") # noqa: LOG015 + logging.info("info message") # noqa: LOG015 + logging.warning("warning message") # noqa: LOG015 + logging.error("error message") # noqa: LOG015 logged = log_file.read_text(encoding="utf-8") @@ -556,7 +556,7 @@ class TestCrawlerLogging: configure_logging() get_crawler(MySpider) - logging.debug("debug message") + logging.debug("debug message") # noqa: LOG015 logged = log_file.read_text(encoding="utf-8") diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 77077e6e5..2dc6817a5 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -15,10 +15,7 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.defer import ( - deferred_f_from_coro_f, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory @@ -46,7 +43,9 @@ class H2DownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: # the import can fail when H2_ENABLED is False - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + from scrapy.core.downloader.handlers.http2 import ( # noqa: PLC0415 + H2DownloadHandler, + ) return H2DownloadHandler diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0775ae1ee..e73bfbccf 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -13,7 +13,7 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal from twisted.internet.defer import inlineCallbacks -from twisted.protocols.ftp import FTPFactory, FTPRealm +from twisted.protocols.ftp import ConnectionLost, FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -27,10 +27,7 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider -from scrapy.utils.defer import ( - deferred_f_from_coro_f, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -184,7 +181,7 @@ class TestS3: @contextlib.contextmanager def _mocked_date(self, date): try: - import botocore.auth # noqa: F401 + import botocore.auth # noqa: F401,PLC0415 except ImportError: yield else: @@ -443,7 +440,6 @@ class TestFTP(TestFTPBase): pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) - from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) meta.update({"ftp_password": "invalid"}) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 3c26b242f..0b3941c09 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -50,6 +50,23 @@ FORMAT = { } +def _skip_if_no_br() -> None: + try: + try: + import brotli # noqa: F401,PLC0415 + except ImportError: + import brotlicffi # noqa: F401,PLC0415 + except ImportError: + pytest.skip("no brotli support") + + +def _skip_if_no_zstd() -> None: + try: + import zstandard # noqa: F401,PLC0415 + except ImportError: + pytest.skip("no zstd support (zstandard)") + + class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) @@ -123,13 +140,8 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", 74837) def test_process_response_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" @@ -143,11 +155,11 @@ class TestHttpCompression: def test_process_response_br_unsupported(self): try: try: - import brotli # noqa: F401 + import brotli # noqa: F401,PLC0415 pytest.skip("Requires not having brotli support") except ImportError: - import brotlicffi # noqa: F401 + import brotlicffi # noqa: F401,PLC0415 pytest.skip("Requires not having brotli support") except ImportError: @@ -176,10 +188,8 @@ class TestHttpCompression: assert newresponse.headers.getlist("Content-Encoding") == [b"br"] def test_process_response_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -198,7 +208,7 @@ class TestHttpCompression: def test_process_response_zstd_unsupported(self): try: - import zstandard # noqa: F401 + import zstandard # noqa: F401,PLC0415 pytest.skip("Requires not having zstandard support") except ImportError: @@ -513,13 +523,8 @@ class TestHttpCompression: mw.process_response(response.request, response, spider) def test_compression_bomb_setting_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -529,10 +534,8 @@ class TestHttpCompression: self._test_compression_bomb_setting("gzip") def test_compression_bomb_setting_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): @@ -549,13 +552,8 @@ class TestHttpCompression: mw.process_response(response.request, response, spider) def test_compression_bomb_spider_attr_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + self._test_compression_bomb_spider_attr("br") def test_compression_bomb_spider_attr_deflate(self): @@ -565,10 +563,8 @@ class TestHttpCompression: self._test_compression_bomb_spider_attr("gzip") def test_compression_bomb_spider_attr_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -583,13 +579,8 @@ class TestHttpCompression: mw.process_response(response.request, response, spider) def test_compression_bomb_request_meta_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -599,10 +590,8 @@ class TestHttpCompression: self._test_compression_bomb_request_meta("gzip") def test_compression_bomb_request_meta_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -632,13 +621,8 @@ class TestHttpCompression: ) def test_download_warnsize_setting_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + self._test_download_warnsize_setting("br") def test_download_warnsize_setting_deflate(self): @@ -648,10 +632,8 @@ class TestHttpCompression: self._test_download_warnsize_setting("gzip") def test_download_warnsize_setting_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -683,13 +665,8 @@ class TestHttpCompression: ) def test_download_warnsize_spider_attr_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + self._test_download_warnsize_spider_attr("br") def test_download_warnsize_spider_attr_deflate(self): @@ -699,10 +676,8 @@ class TestHttpCompression: self._test_download_warnsize_spider_attr("gzip") def test_download_warnsize_spider_attr_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -732,13 +707,8 @@ class TestHttpCompression: ) def test_download_warnsize_request_meta_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - pytest.skip("no brotli") + _skip_if_no_br() + self._test_download_warnsize_request_meta("br") def test_download_warnsize_request_meta_deflate(self): @@ -748,8 +718,6 @@ class TestHttpCompression: self._test_download_warnsize_request_meta("gzip") def test_download_warnsize_request_meta_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - pytest.skip("no zstd support (zstandard)") + _skip_if_no_zstd() + self._test_download_warnsize_request_meta("zstd") diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ce1b7952e..9085af18e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,9 @@ import csv import gzip import json import lzma +import marshal import os +import pickle import random import shutil import string @@ -25,6 +27,7 @@ from urllib.request import pathname2url import lxml.etree import pytest +from packaging.version import Version from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.defer import inlineCallbacks @@ -79,7 +82,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ - from google.cloud.storage import Blob, Bucket, Client + from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 client_mock = mock.create_autospec(Client) @@ -507,7 +510,7 @@ class TestS3FeedStorage(unittest.TestCase): class TestGCSFeedStorage(unittest.TestCase): def test_parse_settings(self): try: - from google.cloud.storage import Client # noqa: F401 + from google.cloud.storage import Client # noqa: F401,PLC0415 except ImportError: pytest.skip("GCSFeedStorage requires google-cloud-storage") @@ -521,7 +524,7 @@ class TestGCSFeedStorage(unittest.TestCase): def test_parse_empty_acl(self): try: - from google.cloud.storage import Client # noqa: F401 + from google.cloud.storage import Client # noqa: F401,PLC0415 except ImportError: pytest.skip("GCSFeedStorage requires google-cloud-storage") @@ -538,7 +541,7 @@ class TestGCSFeedStorage(unittest.TestCase): @deferred_f_from_coro_f async def test_store(self): try: - from google.cloud.storage import Client # noqa: F401 + from google.cloud.storage import Client # noqa: F401,PLC0415 except ImportError: pytest.skip("GCSFeedStorage requires google-cloud-storage") @@ -976,7 +979,6 @@ class TestFeedExport(TestFeedExportBase): ) data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] - import pickle result = self._load_until_eof(data["pickle"], load_func=pickle.load) assert result == expected @@ -997,7 +999,6 @@ class TestFeedExport(TestFeedExportBase): ) data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] - import marshal result = self._load_until_eof(data["marshal"], load_func=marshal.load) assert result == expected @@ -2300,9 +2301,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase): @deferred_f_from_coro_f async def test_exports_compatibility_with_postproc(self): - import marshal - import pickle - filename_to_expected = { self._named_tempfile("csv"): b"foo\r\nbar\r\n", self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', @@ -2484,7 +2482,6 @@ class TestBatchDeliveries(TestFeedExportBase): batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = await self.exported_data(items, settings) - import pickle for batch in data["pickle"]: got_batch = self._load_until_eof(batch, load_func=pickle.load) @@ -2505,7 +2502,6 @@ class TestBatchDeliveries(TestFeedExportBase): batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = await self.exported_data(items, settings) - import marshal for batch in data["marshal"]: got_batch = self._load_until_eof(batch, load_func=marshal.load) @@ -2714,9 +2710,8 @@ class TestBatchDeliveries(TestFeedExportBase): stubs = [] def open(self, *args, **kwargs): - from botocore import __version__ as botocore_version - from botocore.stub import ANY, Stubber - from packaging.version import Version + from botocore import __version__ as botocore_version # noqa: PLC0415 + from botocore.stub import ANY, Stubber # noqa: PLC0415 expected_params = { "Body": ANY, @@ -2912,10 +2907,9 @@ class TestURIParams(ABC): match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", ): crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) else: crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + feed_exporter = crawler.get_extension(FeedExporter) return crawler, feed_exporter def test_default(self): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 80edd50d6..eb9081e7e 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -243,7 +243,7 @@ class TestHttps2ClientProtocol(TestCase): self.conn_closed_deferred = Deferred() - from scrapy.core.http2.protocol import H2ClientFactory + from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415 h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint( @@ -445,7 +445,7 @@ class TestHttps2ClientProtocol(TestCase): def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" - from h2.exceptions import InvalidBodyLengthError + from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415 request = Request(url=self.get_url("/dataloss")) with pytest.raises(ResponseFailed) as exc_info: @@ -525,7 +525,7 @@ class TestHttps2ClientProtocol(TestCase): def assert_inactive_stream(failure): assert failure.check(ResponseFailed) is not None - from scrapy.core.http2.stream import InactiveStreamClosed + from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415 assert any( isinstance(e, InactiveStreamClosed) for e in failure.value.reasons @@ -593,7 +593,7 @@ class TestHttps2ClientProtocol(TestCase): assert str(response.ip_address) == "127.0.0.1" async def _check_invalid_netloc(self, url: str) -> None: - from scrapy.core.http2.stream import InvalidHostname + from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415 request = Request(url) with pytest.raises(InvalidHostname) as exc_info: @@ -630,7 +630,7 @@ class TestHttps2ClientProtocol(TestCase): yield self.make_request_dfd(request) for err in exc_info.value.reasons: - from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415 if isinstance(err, TimeoutError): assert ( diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6bf0b8e3f..90a693bfa 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1441,7 +1441,7 @@ class TestXmlRpcRequest(TestRequest): ) assert r.method == "POST" assert r.encoding == kwargs.get("encoding", "utf-8") - assert r.dont_filter, True + assert r.dont_filter def test_xmlrpc_dumps(self): self._test_request(params=("value",)) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 047f8c610..682428954 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -110,7 +110,7 @@ class TestLogFormatter: assert logkws["level"] == unsupported_value with pytest.raises(TypeError): - logging.log(logkws["level"], "message") + logging.log(logkws["level"], "message") # noqa: LOG015 def test_dropitem_custom_log_level(self): item = {} diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 808fde23d..3d84c7e71 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -5,6 +5,7 @@ import time import warnings from abc import ABC, abstractmethod from datetime import datetime +from ftplib import FTP from io import BytesIO from pathlib import Path from posixpath import split @@ -29,9 +30,7 @@ from scrapy.pipelines.files import ( GCSFilesStore, S3FilesStore, ) -from scrapy.utils.test import ( - get_crawler, -) +from scrapy.utils.test import get_crawler from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func @@ -40,7 +39,7 @@ from .test_pipeline_media import _mocked_download_func def get_gcs_content_and_delete( bucket: Any, path: str ) -> tuple[bytes, list[dict[str, str]], Any]: - from google.cloud import storage + from google.cloud import storage # noqa: PLC0415 client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) bucket = client.get_bucket(bucket) @@ -59,8 +58,6 @@ def get_ftp_content_and_delete( password: str, use_active_mode: bool = False, ) -> bytes: - from ftplib import FTP - ftp = FTP() ftp.connect(host, port) ftp.login(username, password) @@ -553,7 +550,7 @@ class TestS3FilesStore(unittest.TestCase): content_type = "image/png" store = S3FilesStore(uri) - from botocore.stub import Stubber + from botocore.stub import Stubber # noqa: PLC0415 with Stubber(store.s3_client) as stub: stub.add_response( @@ -591,7 +588,7 @@ class TestS3FilesStore(unittest.TestCase): last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) - from botocore.stub import Stubber + from botocore.stub import Stubber # noqa: PLC0415 with Stubber(store.s3_client) as stub: stub.add_response( @@ -650,7 +647,7 @@ class TestGCSFilesStore(unittest.TestCase): already uploaded files. """ try: - import google.cloud.storage # noqa: F401 + import google.cloud.storage # noqa: F401,PLC0415 except ModuleNotFoundError: pytest.skip("google-cloud-storage is not installed") with ( diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 801f88b5a..8875dc04f 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -74,7 +74,7 @@ class TestProxyConnect(TestCase): def setUp(self): try: - import mitmproxy # noqa: F401 + import mitmproxy # noqa: F401,PLC0415 except ImportError: pytest.skip("mitmproxy is not installed") diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 221ccabe6..6a24d2e90 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,21 +1,19 @@ import pytest -from scrapy.robotstxt import decode_robotstxt +from scrapy.robotstxt import ( + ProtegoRobotParser, + PythonRobotParser, + RerpRobotParser, + decode_robotstxt, +) def rerp_available(): # check if robotexclusionrulesparser is installed try: - from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401 - except ImportError: - return False - return True - - -def protego_available(): - # check if protego parser is installed - try: - from protego import Protego # noqa: F401 + from robotexclusionrulesparser import ( # noqa: PLC0415 + RobotExclusionRulesParser, # noqa: F401 + ) except ImportError: return False return True @@ -134,8 +132,6 @@ class TestDecodeRobotsTxt: class TestPythonRobotParser(BaseRobotParserTest): def setup_method(self): - from scrapy.robotstxt import PythonRobotParser - super()._setUp(PythonRobotParser) def test_length_based_precedence(self): @@ -150,19 +146,14 @@ class TestPythonRobotParser(BaseRobotParserTest): @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") class TestRerpRobotParser(BaseRobotParserTest): def setup_method(self): - from scrapy.robotstxt import RerpRobotParser - super()._setUp(RerpRobotParser) def test_length_based_precedence(self): pytest.skip("Rerp does not support length based directives precedence.") -@pytest.mark.skipif(not protego_available(), reason="Protego parser is not installed") class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): - from scrapy.robotstxt import ProtegoRobotParser - super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index b1956ee5c..fa6d11123 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -3,7 +3,9 @@ import warnings def test_deprecated_twisted_version(): with warnings.catch_warnings(record=True) as warns: - from scrapy import twisted_version # pylint: disable=no-name-in-module + from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module + twisted_version, + ) assert twisted_version is not None assert isinstance(twisted_version, tuple) @@ -15,7 +17,7 @@ def test_deprecated_twisted_version(): def test_deprecated_concurrent_requests_per_ip_attribute(): with warnings.catch_warnings(record=True) as warns: - from scrapy.settings.default_settings import ( + from scrapy.settings.default_settings import ( # noqa: PLC0415 CONCURRENT_REQUESTS_PER_IP, ) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 85dc0679e..76609ac52 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -6,6 +6,7 @@ from unittest import mock import pytest +from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -13,6 +14,8 @@ from scrapy.settings import ( SettingsAttribute, get_settings_priority, ) +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler from . import default_settings @@ -447,10 +450,6 @@ class TestSettings: assert mydict["key"] == "val" def test_passing_objects_as_values(self): - from scrapy.core.downloader.handlers.file import FileDownloadHandler - from scrapy.utils.misc import build_from_crawler - from scrapy.utils.test import get_crawler - class TestPipeline: def process_item(self, i, s): return i diff --git a/tests/test_spider.py b/tests/test_spider.py index 7c4b17144..e2843eca4 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,7 +1,9 @@ from __future__ import annotations import gzip +import re import warnings +from datetime import datetime from io import BytesIO from logging import ERROR, WARNING from pathlib import Path @@ -297,8 +299,6 @@ class TestCrawlSpider(TestSpider): ) class _CrawlSpider(self.spider_class): - import re - name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) @@ -635,8 +635,6 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): - from datetime import datetime - for entry in entries: date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") if date_time.year > 2008: @@ -706,8 +704,6 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): - from datetime import datetime - for entry in entries: date_time = datetime.strptime( entry["lastmod"].split("T")[0], "%Y-%m-%d" diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b17332182..895df1804 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse import pytest @@ -33,6 +33,9 @@ from scrapy.spidermiddlewares.referer import ( ) from scrapy.spiders import Spider +if TYPE_CHECKING: + from collections.abc import Sequence + @pytest.fixture def spider() -> Spider: @@ -971,7 +974,7 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii: list[ + scenarii: Sequence[ tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None] ] = [ # type: ignore[assignment] ( @@ -1041,7 +1044,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): request.url, headers={"Location": url}, status=status ) request = cast( - Request, redirectmw.process_response(request, response, spider) + "Request", redirectmw.process_response(request, response, spider) ) referrermw.request_scheduled(request, spider) diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 66a6f5318..b7497c608 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -10,26 +10,26 @@ def test_version_info(): def test_request_shortcut(): - from scrapy.http import FormRequest, Request + from scrapy.http import FormRequest, Request # noqa: PLC0415 assert scrapy.Request is Request assert scrapy.FormRequest is FormRequest def test_spider_shortcut(): - from scrapy.spiders import Spider + from scrapy.spiders import Spider # noqa: PLC0415 assert scrapy.Spider is Spider def test_selector_shortcut(): - from scrapy.selector import Selector + from scrapy.selector import Selector # noqa: PLC0415 assert scrapy.Selector is Selector def test_item_shortcut(): - from scrapy.item import Field, Item + from scrapy.item import Field, Item # noqa: PLC0415 assert scrapy.Item is Item assert scrapy.Field is Field diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 20251ca59..3b1e626c9 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,3 +1,4 @@ +import builtins from io import StringIO from unittest import mock @@ -74,8 +75,6 @@ def test_pformat_windows(isatty, version, terminal_processing): def test_pformat_no_pygments(isatty): isatty.return_value = True - import builtins - real_import = builtins.__import__ def mock_import(name, globals, locals, fromlist, level): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 8600faa8d..5efb29a49 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -24,7 +24,7 @@ def test_iterate_spider_output(): def test_iter_spider_classes(): - import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self + import tests.test_utils_spider # noqa: PLW0406,PLC0415 # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) assert set(it) == {MySpider1, MySpider2} diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index daf09f945..bcbbe74a3 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,4 +1,5 @@ import warnings +from importlib import import_module import pytest @@ -461,8 +462,6 @@ def test_deprecated_imports_from_w3lib(obj_name: str) -> None: obj_type = "attribute" if obj_name == "_safe_chars" else "function" message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead." - from importlib import import_module - getattr(import_module("scrapy.utils.url"), obj_name) assert isinstance(warns[0].message, Warning) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 569f4f639..dd6d7939e 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,11 +18,10 @@ from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util +from twisted.web.client import _makeGetterFactory from scrapy.core.downloader import webclient as client -from scrapy.core.downloader.contextfactory import ( - ScrapyClientContextFactory, -) +from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.http import Headers, Request from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes, to_unicode @@ -51,8 +50,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): f.deferred.addCallback(response_transform or (lambda r: r.body)) return f - from twisted.web.client import _makeGetterFactory - return _makeGetterFactory( to_bytes(url), _clientfactory, diff --git a/tox.ini b/tox.ini index 85935b011..1c1d7c0ec 100644 --- a/tox.ini +++ b/tox.ini @@ -46,17 +46,21 @@ install_command = [testenv:typing] basepython = python3.9 deps = - mypy==1.14.0 - typing-extensions==4.12.2 - types-lxml==2024.12.13 - types-Pygments==2.18.0.20240506 - botocore-stubs==1.35.90 - boto3-stubs[s3]==1.35.90 + mypy==1.16.1 + typing-extensions==4.14.1 + types-defusedxml==0.7.0.20250516 + types-lxml==2025.3.30 + types-pexpect==4.9.0.20250516 + types-Pygments==2.19.0.20250516 + botocore-stubs==1.38.46 + boto3-stubs[s3]==1.39.3 + itemadapter==0.11.0 + Protego==0.5.0 + w3lib==2.3.1 attrs >= 18.2.0 Pillow >= 10.3.0 pyOpenSSL >= 24.2.1 pytest >= 8.2.0 - w3lib >= 2.2.0 commands = mypy {posargs:scrapy tests} @@ -80,14 +84,14 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.3.3 + pylint==3.3.7 commands = pylint conftest.py docs extras scrapy tests [testenv:twinecheck] basepython = python3 deps = - twine==6.0.1 + twine==6.1.0 build==1.2.2.post1 commands = python -m build --sdist