RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bump tool versions. (#6941)

This commit is contained in:
Andrey Rakhmatullin 2025-07-05 13:30:30 +05:00 committed by GitHub
parent 16f168b406
commit df342eee6e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
82 changed files with 382 additions and 492 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.pre-commit-config.yaml
View File

@ -1,8 +1,8 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.9.3
rev: v0.12.2
hooks:
- id: ruff
- id: ruff-check
args: [ --fix ]
- id: ruff-format
- repo: https://github.com/adamchainz/blacken-docs
@ -10,7 +10,7 @@ repos:
hooks:
- id: blacken-docs
additional_dependencies:
- black==24.10.0
- black==25.1.0
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:

6
conftest.py
View File

@ -85,7 +85,7 @@ def requires_uvloop(request):
if not request.node.get_closest_marker("requires_uvloop"):
return
try:
import uvloop
import uvloop # noqa: PLC0415
del uvloop
except ImportError:
@ -97,7 +97,7 @@ def requires_botocore(request):
if not request.node.get_closest_marker("requires_botocore"):
return
try:
import botocore
import botocore # noqa: PLC0415
del botocore
except ImportError:
@ -109,7 +109,7 @@ def requires_boto3(request):
if not request.node.get_closest_marker("requires_boto3"):
return
try:
import boto3
import boto3 # noqa: PLC0415
del boto3
except ImportError:

7
pyproject.toml
View File

@ -106,10 +106,6 @@ follow_imports = "skip"
module = "scrapy.settings.default_settings"
ignore_errors = true
[[tool.mypy.overrides]]
module = "itemadapter"
implicit_reexport = true
[[tool.mypy.overrides]]
module = "twisted"
implicit_reexport = true
@ -394,6 +390,9 @@ banned-module-level-imports = [
"twisted.internet.reactor",
]
[tool.ruff.lint.isort]
split-on-trailing-comma = false
[tool.ruff.lint.per-file-ignores]
# Circular import workarounds
"scrapy/linkextractors/__init__.py" = ["E402"]

6
scrapy/__init__.py
View File

@ -31,11 +31,11 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."
def __getattr__(name: str):
if name == "twisted_version":
import warnings # pylint: disable=reimported
import warnings # noqa: PLC0415 # pylint: disable=reimported
from twisted import version as _txv
from twisted import version as _txv # noqa: PLC0415
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
warnings.warn(
"The scrapy.twisted_version attribute is deprecated, use twisted.version instead",

3
scrapy/commands/bench.py
View File

@ -10,6 +10,7 @@ import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.http import Response, TextResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
import argparse
@ -35,8 +36,6 @@ class Command(ScrapyCommand):
class _BenchServer:
def __enter__(self) -> None:
from scrapy.utils.test import get_testenv
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
self.proc = subprocess.Popen( # noqa: S603
pargs, stdout=subprocess.PIPE, env=get_testenv()

2
scrapy/commands/genspider.py
View File

@ -206,7 +206,7 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(cast(str, spiders_module.__file__)).parent
spiders_dir = Path(cast("str", spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():

6
scrapy/contracts/__init__.py
View File

@ -52,7 +52,7 @@ class Contract:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
return list(cast(Iterable[Any], iterate_spider_output(cb_result)))
return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
request.callback = wrapper
@ -68,7 +68,7 @@ class Contract:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@ -181,7 +181,7 @@ class ContractsManager:
def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
try:
output = cb(response, **cb_kwargs)
output = list(cast(Iterable[Any], iterate_spider_output(output)))
output = list(cast("Iterable[Any]", iterate_spider_output(output)))
except Exception:
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())

2
scrapy/core/downloader/__init__.py
View File

@ -162,7 +162,7 @@ class Downloader:
def get_slot_key(self, request: Request) -> str:
if self.DOWNLOAD_SLOT in request.meta:
return cast(str, request.meta[self.DOWNLOAD_SLOT])
return cast("str", request.meta[self.DOWNLOAD_SLOT])
key = urlparse_cached(request).hostname or ""
if self.ip_concurrency:

3
scrapy/core/downloader/handlers/http11.py
View File

@ -386,6 +386,7 @@ class ScrapyAgent:
if not proxy_port:
proxy_port = 443 if proxy_parsed.scheme == "https" else 80
if urlparse_cached(request).scheme == "https":
assert proxy_host is not None
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxy_host, proxy_port, proxyAuth)
return self._TunnelingAgent(
@ -430,7 +431,7 @@ class ScrapyAgent:
method,
to_bytes(url, encoding="ascii"),
headers,
cast(IBodyProducer, bodyproducer),
cast("IBodyProducer", bodyproducer),
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)

6
scrapy/core/downloader/handlers/s3.py
View File

@ -51,8 +51,8 @@ class S3DownloadHandler:
self.anon = kw.get("anon")
self._signer = None
import botocore.auth
import botocore.credentials
import botocore.auth # noqa: PLC0415
import botocore.credentials # noqa: PLC0415
kw.pop("anon", None)
if kw:
@ -87,7 +87,7 @@ class S3DownloadHandler:
if self.anon:
request = request.replace(url=url)
else:
import botocore.awsrequest
import botocore.awsrequest # noqa: PLC0415
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,

9
scrapy/core/downloader/middleware.py
View File

@ -6,7 +6,6 @@ See documentation in docs/topics/downloader-middleware.rst
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, cast
from twisted.internet.defer import Deferred, inlineCallbacks
@ -18,7 +17,7 @@ from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import _defer_sleep, deferred_from_coro
if TYPE_CHECKING:
from collections.abc import Generator
from collections.abc import Callable, Generator
from scrapy import Spider
from scrapy.settings import BaseSettings
@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
request: Request,
) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_request"]:
method = cast(Callable, method)
method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, spider=spider)
)
@ -76,7 +75,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
for method in self.methods["process_response"]:
method = cast(Callable, method)
method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, response=response, spider=spider)
)
@ -94,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
exception: Exception,
) -> Generator[Deferred[Any], Any, Response | Request]:
for method in self.methods["process_exception"]:
method = cast(Callable, method)
method = cast("Callable", method)
response = yield deferred_from_coro(
method(request=request, exception=exception, spider=spider)
)

11
scrapy/core/engine.py
View File

@ -17,13 +17,11 @@ from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, su
from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.scheduler import BaseScheduler
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
create_looping_call,
)
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
@ -39,7 +37,6 @@ if TYPE_CHECKING:
from twisted.internet.task import LoopingCall
from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import BaseScheduler
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
@ -123,8 +120,6 @@ class ExecutionEngine:
raise
def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]:
from scrapy.core.scheduler import BaseScheduler
scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
@ -506,7 +501,7 @@ class ExecutionEngine:
dfd.addErrback(log_failure("Scraper close failure"))
if hasattr(self._slot.scheduler, "close"):
dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason))
dfd.addBoth(lambda _: cast("_Slot", self._slot).scheduler.close(reason))
dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(

2
scrapy/core/scheduler.py
View File

@ -491,7 +491,7 @@ class Scheduler(BaseScheduler):
if not path.exists():
return []
with path.open(encoding="utf-8") as f:
return cast(list[int], json.load(f))
return cast("list[int]", json.load(f))
def _write_dqs_state(self, dqdir: str, state: list[int]) -> None:
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:

10
scrapy/core/spidermw.py
View File

@ -138,7 +138,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
) -> Deferred[Iterable[_T] | AsyncIterator[_T]]:
for method in self.methods["process_spider_input"]:
method = cast(Callable, method)
method = cast("Callable", method)
try:
result = method(response=response, spider=spider)
if result is not None:
@ -166,7 +166,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield from iterable
except Exception as ex:
exception_result = cast(
Union[Failure, MutableChain[_T]],
"Union[Failure, MutableChain[_T]]",
self._process_spider_exception(
response, spider, ex, exception_processor_index
),
@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield r
except Exception as ex:
exception_result = cast(
Union[Failure, MutableAsyncChain[_T]],
"Union[Failure, MutableAsyncChain[_T]]",
self._process_spider_exception(
response, spider, ex, exception_processor_index
),
@ -212,7 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
method = cast(Callable, method)
method = cast("Callable", method)
result = method(response=response, exception=exception, spider=spider)
if _isiterable(result):
# stop exception handling by handing control over to the
@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast(
Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
"Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result
)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture

6
scrapy/crawler.py
View File

@ -8,11 +8,7 @@ import signal
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, TypeVar
from twisted.internet.defer import (
Deferred,
DeferredList,
inlineCallbacks,
)
from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
from scrapy import Spider, signals
from scrapy.addons import AddonManager

2
scrapy/downloadermiddlewares/redirect.py
View File

@ -202,6 +202,6 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
redirected = self._redirect_request_using_get(request, url)
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
if cast(float, interval) < self._maxdelay:
if cast("float", interval) < self._maxdelay:
return self._redirect(redirected, request, spider, "meta refresh")
return response

8
scrapy/extensions/feedexport.py
View File

@ -218,7 +218,7 @@ class S3FeedStorage(BlockingFeedStorage):
region_name: str | None = None,
):
try:
import boto3.session
import boto3.session # noqa: PLC0415
except ImportError:
raise NotConfigured("missing boto3 library")
u = urlparse(uri)
@ -317,7 +317,7 @@ class GCSFeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
from google.cloud.storage import Client
from google.cloud.storage import Client # noqa: PLC0415
client = Client(project=self.project_id)
bucket = client.get_bucket(self.bucket_name)
@ -413,7 +413,7 @@ class FeedSlot:
self.file = self.storage.open(self.spider)
if "postprocessing" in self.feed_options:
self.file = cast(
IO[bytes],
"IO[bytes]",
PostProcessingManager(
self.feed_options["postprocessing"],
self.file,
@ -662,7 +662,7 @@ class FeedExporter:
def _load_components(self, setting_prefix: str) -> dict[str, Any]:
conf = without_none_values(
cast(dict[str, str], self.settings.getwithbase(setting_prefix))
cast("dict[str, str]", self.settings.getwithbase(setting_prefix))
)
d = {}
for k, v in conf.items():

4
scrapy/extensions/httpcache.py
View File

@ -307,7 +307,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return None # expired
return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301
return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"])) # noqa: S301
class FilesystemCacheStorage:
@ -389,7 +389,7 @@ class FilesystemCacheStorage:
if 0 < self.expiration_secs < time() - mtime:
return None # expired
with self._open(metapath, "rb") as f:
return cast(dict[str, Any], pickle.load(f)) # noqa: S301
return cast("dict[str, Any]", pickle.load(f)) # noqa: S301
def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:

5
scrapy/extensions/logstats.py
View File

@ -5,10 +5,7 @@ from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
create_looping_call,
)
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
if TYPE_CHECKING:
from twisted.internet.task import LoopingCall

5
scrapy/extensions/memusage.py
View File

@ -16,10 +16,7 @@ from typing import TYPE_CHECKING
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
create_looping_call,
)
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.engine import get_engine_status
if TYPE_CHECKING:

5
scrapy/extensions/periodic_log.py
View File

@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Any
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
create_looping_call,
)
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:

2
scrapy/extensions/postprocessing.py
View File

@ -142,7 +142,7 @@ class PostProcessingManager(IOBase):
:return: returns number of bytes written
:rtype: int
"""
return cast(int, self.head_plugin.write(data))
return cast("int", self.head_plugin.write(data))
def tell(self) -> int:
return self.file.tell()

7
scrapy/extensions/telnet.py
View File

@ -12,6 +12,8 @@ import os
import pprint
from typing import TYPE_CHECKING, Any
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
from twisted.internet import protocol
from scrapy import signals
@ -22,7 +24,6 @@ from scrapy.utils.reactor import listen_tcp
from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
from twisted.conch import telnet
from twisted.internet.tcp import Port
# typing.Self requires Python 3.11
@ -76,10 +77,6 @@ class TelnetConsole(protocol.ServerFactory):
self.port.stopListening()
def protocol(self) -> telnet.TelnetTransport:
# these import twisted.internet.reactor
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
class Portal:
"""An implementation of IPortal"""

4
scrapy/http/cookies.py
View File

@ -161,7 +161,7 @@ class WrappedRequest:
HTML document, and the user had no option to approve the automatic
fetching of the image, this should be true.
"""
return cast(bool, self.request.meta.get("is_unverifiable", False))
return cast("bool", self.request.meta.get("is_unverifiable", False))
@property
def full_url(self) -> str:
@ -181,7 +181,7 @@ class WrappedRequest:
@property
def origin_req_host(self) -> str:
return cast(str, urlparse_cached(self.request).hostname)
return cast("str", urlparse_cached(self.request).hostname)
def has_header(self, name: str) -> bool:
return name in self.request.headers

6
scrapy/http/headers.py
View File

@ -69,19 +69,19 @@ class Headers(CaselessDict):
def __getitem__(self, key: AnyStr) -> bytes | None:
try:
return cast(list[bytes], super().__getitem__(key))[-1]
return cast("list[bytes]", super().__getitem__(key))[-1]
except IndexError:
return None
def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
try:
return cast(list[bytes], super().get(key, def_val))[-1]
return cast("list[bytes]", super().get(key, def_val))[-1]
except IndexError:
return None
def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
try:
return cast(list[bytes], super().__getitem__(key))
return cast("list[bytes]", super().__getitem__(key))
except KeyError:
if def_val is not None:
return self.normvalue(def_val)

31
scrapy/http/request/form.py
View File

@ -11,13 +11,7 @@ from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from lxml.html import (
FormElement,
InputElement,
MultipleSelectOptions,
SelectElement,
TextareaElement,
)
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
@ -25,6 +19,13 @@ from scrapy.utils.python import is_listlike, to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from lxml.html import (
FormElement,
InputElement,
MultipleSelectOptions,
SelectElement,
TextareaElement,
)
from typing_extensions import Self
from scrapy.http.response.text import TextResponse
@ -76,8 +77,6 @@ class FormRequest(Request):
kwargs.setdefault("encoding", response.encoding)
if formcss is not None:
from parsel.csstranslator import HTMLTranslator
formxpath = HTMLTranslator().css_to_xpath(formcss)
form = _get_form(response, formname, formid, formnumber, formxpath)
@ -107,7 +106,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
values = [
(to_bytes(k, enc), to_bytes(v, enc))
for k, vs in seq
for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)])
]
return urlencode(values, doseq=True)
@ -128,12 +127,12 @@ def _get_form(
if formname is not None:
f = root.xpath(f'//form[@name="{formname}"]')
if f:
return cast(FormElement, f[0])
return cast("FormElement", f[0])
if formid is not None:
f = root.xpath(f'//form[@id="{formid}"]')
if f:
return cast(FormElement, f[0])
return cast("FormElement", f[0])
# Get form element from xpath, if not found, go up
if formxpath is not None:
@ -142,7 +141,7 @@ def _get_form(
el = nodes[0]
while True:
if el.tag == "form":
return cast(FormElement, el)
return cast("FormElement", el)
el = el.getparent()
if el is None:
break
@ -153,7 +152,7 @@ def _get_form(
form = forms[formnumber]
except IndexError:
raise IndexError(f"Form number {formnumber} not found in {response}")
return cast(FormElement, form)
return cast("FormElement", form)
def _get_inputs(
@ -201,7 +200,7 @@ def _value(
n = ele.name
v = ele.value
if ele.tag == "select":
return _select_value(cast(SelectElement, ele), n, v)
return _select_value(cast("SelectElement", ele), n, v)
return n, v
@ -251,7 +250,7 @@ def _get_clickable(
except IndexError:
pass
else:
return (el.get("name"), el.get("value") or "")
return (cast("str", el.get("name")), el.get("value") or "")
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such

21
scrapy/http/response/text.py
View File

@ -104,13 +104,14 @@ class TextResponse(Response):
@memoizemethod_noargs
def _headers_encoding(self) -> str | None:
content_type = cast(bytes, self.headers.get(b"Content-Type", b""))
content_type = cast("bytes", self.headers.get(b"Content-Type", b""))
return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
def _body_inferred_encoding(self) -> str:
if self._cached_benc is None:
content_type = to_unicode(
cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1"
cast("bytes", self.headers.get(b"Content-Type", b"")),
encoding="latin-1",
)
benc, ubody = html_to_unicode(
content_type,
@ -141,31 +142,25 @@ class TextResponse(Response):
@property
def selector(self) -> Selector:
from scrapy.selector import Selector
# circular import
from scrapy.selector import Selector # noqa: PLC0415
if self._cached_selector is None:
self._cached_selector = Selector(self)
return self._cached_selector
def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
from scrapy.selector import SelectorList
if not hasattr(self.selector, "jmespath"):
raise AttributeError(
"Please install parsel >= 1.8.1 to get jmespath support"
)
return cast(SelectorList, self.selector.jmespath(query, **kwargs))
return cast("SelectorList", self.selector.jmespath(query, **kwargs))
def xpath(self, query: str, **kwargs: Any) -> SelectorList:
from scrapy.selector import SelectorList
return cast(SelectorList, self.selector.xpath(query, **kwargs))
return cast("SelectorList", self.selector.xpath(query, **kwargs))
def css(self, query: str) -> SelectorList:
from scrapy.selector import SelectorList
return cast(SelectorList, self.selector.css(query))
return cast("SelectorList", self.selector.css(query))
def follow(
self,

6
scrapy/linkextractors/lxmlhtml.py
View File

@ -71,12 +71,12 @@ class LxmlParserLinkExtractor:
self.scan_tag: Callable[[str], bool] = (
tag
if callable(tag)
else cast(Callable[[str], bool], partial(operator.eq, tag))
else cast("Callable[[str], bool]", partial(operator.eq, tag))
)
self.scan_attr: Callable[[str], bool] = (
attr
if callable(attr)
else cast(Callable[[str], bool], partial(operator.eq, attr))
else cast("Callable[[str], bool]", partial(operator.eq, attr))
)
self.process_attr: Callable[[Any], Any] = (
process if callable(process) else _identity
@ -84,7 +84,7 @@ class LxmlParserLinkExtractor:
self.unique: bool = unique
self.strip: bool = strip
self.link_key: Callable[[Link], str] = (
cast(Callable[[Link], str], operator.attrgetter("url"))
cast("Callable[[Link], str]", operator.attrgetter("url"))
if canonicalized
else _canonicalize_link_url
)

3
scrapy/mail.py
View File

@ -223,7 +223,8 @@ class MailSender:
def _create_sender_factory(
self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any]
) -> ESMTPSenderFactory:
from twisted.mail.smtp import ESMTPSenderFactory
# imports twisted.internet.reactor
from twisted.mail.smtp import ESMTPSenderFactory # noqa: PLC0415
factory_keywords: dict[str, Any] = {
"heloFallback": True,

2
scrapy/middleware.py
View File

@ -63,7 +63,7 @@ class MiddlewareManager(ABC):
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return cast(_T, instance)
return cast("_T", instance)
@classmethod
def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:

16
scrapy/pipelines/files.py
View File

@ -169,7 +169,7 @@ class S3FilesStore:
def __init__(self, uri: str):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
import botocore.session
import botocore.session # noqa: PLC0415
session = botocore.session.get_session()
self.s3_client = session.create_client(
@ -284,7 +284,7 @@ class GCSFilesStore:
POLICY = None
def __init__(self, uri: str):
from google.cloud import storage
from google.cloud import storage # noqa: PLC0415
client = storage.Client(project=self.GCS_PROJECT_ID)
bucket, prefix = uri[5:].split("/", 1)
@ -317,7 +317,7 @@ class GCSFilesStore:
blob_path = self._get_blob_path(path)
return cast(
Deferred[StatInfo],
"Deferred[StatInfo]",
deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),
)
@ -531,7 +531,9 @@ class FilesPipeline(MediaPipeline):
@classmethod
def _update_stores(cls, settings: BaseSettings) -> None:
s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])
s3store: type[S3FilesStore] = cast(
"type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
)
s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
@ -542,13 +544,13 @@ class FilesPipeline(MediaPipeline):
s3store.POLICY = settings["FILES_STORE_S3_ACL"]
gcs_store: type[GCSFilesStore] = cast(
type[GCSFilesStore], cls.STORE_SCHEMES["gs"]
"type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
)
gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None
ftp_store: type[FTPFilesStore] = cast(
type[FTPFilesStore], cls.STORE_SCHEMES["ftp"]
"type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"]
)
ftp_store.FTP_USERNAME = settings["FTP_USER"]
ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
@ -742,5 +744,5 @@ class FilesPipeline(MediaPipeline):
media_ext = ""
media_type = mimetypes.guess_type(request.url)[0]
if media_type:
media_ext = cast(str, mimetypes.guess_extension(media_type))
media_ext = cast("str", mimetypes.guess_extension(media_type))
return f"full/{media_guid}{media_ext}"

2
scrapy/pipelines/images.py
View File

@ -63,7 +63,7 @@ class ImagesPipeline(FilesPipeline):
crawler: Crawler | None = None,
):
try:
from PIL import Image
from PIL import Image # noqa: PLC0415
self._Image = Image
except ImportError:

4
scrapy/pqueues.py
View File

@ -4,7 +4,6 @@ import hashlib
import logging
from typing import TYPE_CHECKING, Protocol, cast
from scrapy import Request
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
@ -13,6 +12,7 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request
from scrapy.core.downloader import Downloader
from scrapy.crawler import Crawler
@ -211,7 +211,7 @@ class ScrapyPriorityQueue:
except KeyError:
queue = self.queues[self.curprio]
# Protocols can't declare optional members
return cast(Request, queue.peek()) # type: ignore[attr-defined]
return cast("Request", queue.peek()) # type: ignore[attr-defined]
def close(self) -> list[int]:
active: set[int] = set()

9
scrapy/robotstxt.py
View File

@ -4,6 +4,9 @@ import logging
import sys
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING
from urllib.robotparser import RobotFileParser
from protego import Protego
from scrapy.utils.python import to_unicode
@ -67,8 +70,6 @@ class RobotParser(metaclass=ABCMeta):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
from urllib.robotparser import RobotFileParser
self.spider: Spider | None = spider
body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp: RobotFileParser = RobotFileParser()
@ -87,7 +88,7 @@ class PythonRobotParser(RobotParser):
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
from robotexclusionrulesparser import RobotExclusionRulesParser
from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: PLC0415
self.spider: Spider | None = spider
self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser()
@ -107,8 +108,6 @@ class RerpRobotParser(RobotParser):
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
from protego import Protego
self.spider: Spider | None = spider
body_decoded = decode_robotstxt(robotstxt_body, spider)
self.rp = Protego.parse(body_decoded)

8
scrapy/settings/__init__.py
View File

@ -336,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
stored.
"""
if len(self) > 0:
return max(cast(int, self.getpriority(name)) for name in self)
return max(cast("int", self.getpriority(name)) for name in self)
return get_settings_priority("default")
def replace_in_component_priority_dict(
@ -519,11 +519,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"""
self._assert_mutability()
if isinstance(values, str):
values = cast(dict[_SettingsKeyT, Any], json.loads(values))
values = cast("dict[_SettingsKeyT, Any]", json.loads(values))
if values is not None:
if isinstance(values, BaseSettings):
for name, value in values.items():
self.set(name, value, cast(int, values.getpriority(name)))
self.set(name, value, cast("int", values.getpriority(name)))
else:
for name, value in values.items():
self.set(name, value, priority)
@ -533,7 +533,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
raise KeyError(name)
self._assert_mutability()
priority = get_settings_priority(priority)
if priority >= cast(int, self.getpriority(name)):
if priority >= cast("int", self.getpriority(name)):
del self.attributes[name]
def __delitem__(self, name: _SettingsKeyT) -> None:

4
scrapy/settings/default_settings.py
View File

@ -535,9 +535,9 @@ WARN_ON_GENERATOR_RETURN_VALUE = True
def __getattr__(name: str):
if name == "CONCURRENT_REQUESTS_PER_IP":
import warnings
import warnings # noqa: PLC0415
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
warnings.warn(
"The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",

3
scrapy/shell.py
View File

@ -16,6 +16,7 @@ from twisted.internet import defer, threads
from twisted.python import threadable
from w3lib.url import any_to_uri
import scrapy
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
@ -164,8 +165,6 @@ class Shell:
request: Request | None = None,
spider: Spider | None = None,
) -> None:
import scrapy
self.vars["scrapy"] = scrapy
self.vars["crawler"] = self.crawler
self.vars["item"] = self.item_class()

2
scrapy/spidermiddlewares/referer.py
View File

@ -315,7 +315,7 @@ def _load_policy_class(
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
return cast(type[ReferrerPolicy], load_object(policy))
return cast("type[ReferrerPolicy]", load_object(policy))
except ValueError:
tokens = [token.strip() for token in policy.lower().split(",")]
# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header

3
scrapy/spiders/__init__.py
View File

@ -55,7 +55,8 @@ class Spider(object_ref):
@property
def logger(self) -> SpiderLoggerAdapter:
from scrapy.utils.log import SpiderLoggerAdapter
# circular import
from scrapy.utils.log import SpiderLoggerAdapter # noqa: PLC0415
logger = logging.getLogger(self.name)
return SpiderLoggerAdapter(logger, {"spider": self})

22
scrapy/spiders/crawl.py
View File

@ -12,8 +12,6 @@ import warnings
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
from twisted.python.failure import Failure
from scrapy.http import HtmlResponse, Request, Response
from scrapy.link import Link
from scrapy.linkextractors import LinkExtractor
@ -26,6 +24,8 @@ from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
@ -81,12 +81,14 @@ class Rule:
def _compile(self, spider: Spider) -> None:
# this replaces method names with methods and we can't express this in type hints
self.callback = cast("CallbackT", _get_method(self.callback, spider))
self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
self.errback = cast(
"Callable[[Failure], Any]", _get_method(self.errback, spider)
)
self.process_links = cast(
ProcessLinksT, _get_method(self.process_links, spider)
"ProcessLinksT", _get_method(self.process_links, spider)
)
self.process_request = cast(
ProcessRequestT, _get_method(self.process_request, spider)
"ProcessRequestT", _get_method(self.process_request, spider)
)
@ -141,13 +143,13 @@ class CrawlSpider(Spider):
for lnk in rule.link_extractor.extract_links(response)
if lnk not in seen
]
for link in cast(ProcessLinksT, rule.process_links)(links):
for link in cast("ProcessLinksT", rule.process_links)(links):
seen.add(link)
request = self._build_request(rule_index, link)
yield cast(ProcessRequestT, rule.process_request)(request, response)
yield cast("ProcessRequestT", rule.process_request)(request, response)
def _callback(self, response: Response, **cb_kwargs: Any) -> Any:
rule = self._rules[cast(int, response.meta["rule"])]
rule = self._rules[cast("int", response.meta["rule"])]
return self.parse_with_rules(
response,
cast("CallbackT", rule.callback),
@ -156,9 +158,9 @@ class CrawlSpider(Spider):
)
def _errback(self, failure: Failure) -> Iterable[Any]:
rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined]
rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined]
return self._handle_failure(
failure, cast(Callable[[Failure], Any], rule.errback)
failure, cast("Callable[[Failure], Any]", rule.errback)
)
async def parse_with_rules(

7
scrapy/spiders/init.py
View File

@ -1,15 +1,16 @@
from __future__ import annotations
import warnings
from collections.abc import AsyncIterator, Iterable
from typing import TYPE_CHECKING, Any, cast
from scrapy import Request
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spiders import Spider
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
from scrapy import Request
from scrapy.http import Response
@ -39,7 +40,7 @@ class InitSpider(Spider):
def start_requests(self) -> Iterable[Request]:
self._postinit_reqs: Iterable[Request] = super().start_requests()
return cast(Iterable[Request], iterate_spider_output(self.init_request()))
return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
def initialized(self, response: Response | None = None) -> Any:
"""This method must be set as the callback of your last initialization

2
scrapy/utils/boto.py
View File

@ -3,7 +3,7 @@
def is_botocore_available() -> bool:
try:
import botocore # noqa: F401
import botocore # noqa: F401,PLC0415
return True
except ImportError:

2
scrapy/utils/conf.py
View File

@ -153,7 +153,7 @@ def feed_process_params_from_cli(
suitable to be used as the FEEDS setting.
"""
valid_output_formats: Iterable[str] = without_none_values(
cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS"))
cast("dict[str, str]", settings.getwithbase("FEED_EXPORTERS"))
).keys()
def check_valid_format(output_format: str) -> None:

19
scrapy/utils/console.py
View File

@ -1,5 +1,6 @@
from __future__ import annotations
import code
from collections.abc import Callable
from functools import wraps
from typing import TYPE_CHECKING, Any
@ -16,13 +17,13 @@ def _embed_ipython_shell(
) -> EmbedFuncT:
"""Start an IPython Shell"""
try:
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100
from IPython.terminal.ipapp import load_default_config
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
except ImportError:
from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100
from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100,PLC0415
InteractiveShellEmbed,
)
from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef]
from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] # noqa: PLC0415
load_default_config,
)
@ -46,7 +47,7 @@ def _embed_bpython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
) -> EmbedFuncT:
"""Start a bpython shell"""
import bpython
import bpython # noqa: PLC0415
@wraps(_embed_bpython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
@ -59,7 +60,7 @@ def _embed_ptpython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
) -> EmbedFuncT:
"""Start a ptpython shell"""
import ptpython.repl # pylint: disable=import-error
import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error
@wraps(_embed_ptpython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
@ -73,14 +74,12 @@ def _embed_standard_shell(
namespace: dict[str, Any] = {}, banner: str = ""
) -> EmbedFuncT:
"""Start a standard python shell"""
import code
try: # readline module is only available on unix systems
import readline
import readline # noqa: PLC0415
except ImportError:
pass
else:
import rlcompleter # noqa: F401
import rlcompleter # noqa: F401,PLC0415
readline.parse_and_bind("tab:complete") # type: ignore[attr-defined]

3
scrapy/utils/datatypes.py
View File

@ -32,7 +32,8 @@ class CaselessDict(dict):
__slots__ = ()
def __new__(cls, *args: Any, **kwargs: Any) -> Self:
from scrapy.http.headers import Headers
# circular import
from scrapy.http.headers import Headers # noqa: PLC0415
if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
warnings.warn(

9
scrapy/utils/defer.py
View File

@ -12,12 +12,7 @@ from collections.abc import Awaitable, Coroutine, Iterable, Iterator
from functools import wraps
from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload
from twisted.internet.defer import (
Deferred,
DeferredList,
fail,
succeed,
)
from twisted.internet.defer import Deferred, DeferredList, fail, succeed
from twisted.internet.task import Cooperator
from twisted.python import failure
@ -414,7 +409,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2:
if not is_asyncio_available():
# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
# that use asyncio, e.g. "await asyncio.sleep(1)"
return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o))
return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o))
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
return Deferred.fromFuture(asyncio.ensure_future(o))
return o

6
scrapy/utils/display.py
View File

@ -34,11 +34,11 @@ def _colorize(text: str, colorize: bool = True) -> str:
if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
return text
try:
from pygments import highlight
from pygments import highlight # noqa: PLC0415
except ImportError:
return text
from pygments.formatters import TerminalFormatter
from pygments.lexers import PythonLexer
from pygments.formatters import TerminalFormatter # noqa: PLC0415
from pygments.lexers import PythonLexer # noqa: PLC0415
return highlight(text, PythonLexer(), TerminalFormatter())

4
scrapy/utils/iterators.py
View File

@ -147,12 +147,12 @@ class _StreamReader:
def _read_string(self, n: int = 65535) -> bytes:
s, e = self._ptr, self._ptr + n
self._ptr = e
return cast(bytes, self._text)[s:e]
return cast("bytes", self._text)[s:e]
def _read_unicode(self, n: int = 65535) -> bytes:
s, e = self._ptr, self._ptr + n
self._ptr = e
return cast(str, self._text)[s:e].encode("utf-8")
return cast("str", self._text)[s:e].encode("utf-8")
def csviter(

10
scrapy/utils/log.py
View File

@ -5,9 +5,9 @@ import pprint
import sys
from collections.abc import MutableMapping
from logging.config import dictConfig
from types import TracebackType
from typing import TYPE_CHECKING, Any, Optional, cast
from twisted.internet import asyncioreactor
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
@ -16,6 +16,8 @@ from scrapy.settings import Settings, _SettingsKeyT
from scrapy.utils.versions import get_versions
if TYPE_CHECKING:
from types import TracebackType
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatterResult
@ -33,7 +35,7 @@ def failure_to_exc_info(
return (
failure.type,
failure.value,
cast(Optional[TracebackType], failure.getTracebackObject()),
cast("Optional[TracebackType]", failure.getTracebackObject()),
)
return None
@ -182,7 +184,7 @@ def log_scrapy_info(settings: Settings) -> None:
def log_reactor_info() -> None:
from twisted.internet import asyncioreactor, reactor
from twisted.internet import reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
@ -240,7 +242,7 @@ def logformatter_adapter(
message = logkws.get("msg") or ""
# NOTE: This also handles 'args' being an empty dict, that case doesn't
# play well in logger.log calls
args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"]
args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"]
return (level, message, args)

7
scrapy/utils/misc.py
View File

@ -9,7 +9,6 @@ import os
import re
import warnings
from collections import deque
from collections.abc import Iterable
from contextlib import contextmanager
from functools import partial
from importlib import import_module
@ -21,7 +20,7 @@ from scrapy.item import Item
from scrapy.utils.datatypes import LocalWeakReferencedCache
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterable, Iterator
from types import ModuleType
from scrapy import Spider
@ -41,7 +40,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
if arg is None:
return []
if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
return cast(Iterable[Any], arg)
return cast("Iterable[Any]", arg)
return [arg]
@ -202,7 +201,7 @@ def build_from_crawler(
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return cast(T, instance)
return cast("T", instance)
@contextmanager

13
scrapy/utils/reactor.py
View File

@ -29,7 +29,7 @@ if TYPE_CHECKING:
_T = TypeVar("_T")
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503
"""Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor
@ -39,7 +39,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
return reactor.listenTCP(0, factory, interface=host)
if len(portrange) == 1:
return reactor.listenTCP(portrange[0], factory, interface=host)
for x in range(portrange[0], portrange[1] + 1): # noqa: RET503
for x in range(portrange[0], portrange[1] + 1):
try:
return reactor.listenTCP(x, factory, interface=host)
except error.CannotListenError:
@ -60,7 +60,8 @@ class CallLaterOnce(Generic[_T]):
self._deferreds: list[Deferred] = []
def schedule(self, delay: float = 0) -> None:
from scrapy.utils.asyncio import call_later
# circular import
from scrapy.utils.asyncio import call_later # noqa: PLC0415
if self._call is None:
self._call = call_later(delay, self)
@ -70,7 +71,8 @@ class CallLaterOnce(Generic[_T]):
self._call.cancel()
def __call__(self) -> _T:
from scrapy.utils.asyncio import call_later
# circular import
from scrapy.utils.asyncio import call_later # noqa: PLC0415
self._call = None
result = self._func(*self._a, **self._kw)
@ -82,7 +84,8 @@ class CallLaterOnce(Generic[_T]):
return result
async def wait(self):
from scrapy.utils.defer import maybe_deferred_to_future
# circular import
from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415
d = Deferred()
self._deferreds.append(d)

3
scrapy/utils/response.py
View File

@ -91,7 +91,8 @@ def open_in_browser(
if "item name" not in response.body:
open_in_browser(response)
"""
from scrapy.http import HtmlResponse, TextResponse
# circular imports
from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415
# XXX: this implementation is a bit dirty and could be improved
body = response.body

2
scrapy/utils/sitemap.py
View File

@ -24,7 +24,7 @@ class Sitemap:
xmlp = lxml.etree.XMLParser(
recover=True, remove_comments=True, resolve_entities=False
)
self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320
self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
rt = self._root.tag
assert isinstance(rt, str)
self.type = rt.split("}", 1)[1] if "}" in rt else rt

12
scrapy/utils/test.py
View File

@ -7,6 +7,7 @@ from __future__ import annotations
import asyncio
import os
import warnings
from ftplib import FTP
from importlib import import_module
from pathlib import Path
from posixpath import split
@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, cast
from unittest import TestCase, mock
from twisted.trial.unittest import SkipTest
from twisted.web.client import Agent
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.deprecate import create_deprecated_class
@ -59,7 +62,7 @@ def skip_if_no_boto() -> None:
def get_gcs_content_and_delete(
bucket: Any, path: str
) -> tuple[bytes, list[dict[str, str]], Any]:
from google.cloud import storage
from google.cloud import storage # noqa: PLC0415
warnings.warn(
"The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
@ -83,8 +86,6 @@ def get_ftp_content_and_delete(
password: str,
use_active_mode: bool = False,
) -> bytes:
from ftplib import FTP
warnings.warn(
"The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
@ -137,8 +138,6 @@ def get_crawler(
will be used to populate the crawler settings with a project level
priority.
"""
from scrapy.crawler import CrawlerRunner
# When needed, useful settings can be added here, e.g. ones that prevent
# deprecation warnings.
settings: dict[str, Any] = {
@ -192,7 +191,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.
"""
from google.cloud.storage import Blob, Bucket, Client
from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415
warnings.warn(
"The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",
@ -213,7 +212,6 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
from twisted.internet import reactor
from twisted.web.client import Agent # imports twisted.internet.reactor
agent = Agent(reactor)
return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))

6
scrapy/utils/testproc.py
View File

@ -6,7 +6,6 @@ import warnings
from typing import TYPE_CHECKING, cast
from twisted.internet.defer import Deferred
from twisted.internet.error import ProcessTerminated
from twisted.internet.protocol import ProcessProtocol
from scrapy.exceptions import ScrapyDeprecationWarning
@ -14,6 +13,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable
from twisted.internet.error import ProcessTerminated
from twisted.python.failure import Failure
@ -55,7 +55,7 @@ class ProcessTest:
msg += "\n"
msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
raise RuntimeError(msg)
return cast(int, pp.exitcode), pp.out, pp.err
return cast("int", pp.exitcode), pp.out, pp.err
class TestProcessProtocol(ProcessProtocol):
@ -72,5 +72,5 @@ class TestProcessProtocol(ProcessProtocol):
self.err += data
def processEnded(self, status: Failure) -> None:
self.exitcode = cast(ProcessTerminated, status.value).exitCode
self.exitcode = cast("ProcessTerminated", status.value).exitCode
self.deferred.callback(self)

2
tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
View File

@ -21,7 +21,7 @@ def log_task_exception(task: Task) -> None:
try:
task.result()
except Exception:
logging.exception("Crawl task failed")
logging.exception("Crawl task failed") # noqa: LOG015
process = AsyncCrawlerProcess()

150
tests/test_command_genspider.py
View File

@ -3,6 +3,8 @@ from __future__ import annotations
import os
from pathlib import Path
import pytest
from tests.test_commands import TestCommandBase, TestProjectBase
@ -15,7 +17,16 @@ class TestGenspiderCommand(TestCommandBase):
assert self.call("genspider", "test_name", "test.com") == 0
assert Path(self.proj_mod_path, "spiders", "test_name.py").exists()
def test_template(self, tplname="crawl"):
@pytest.mark.parametrize(
"tplname",
[
"basic",
"crawl",
"xmlfeed",
"csvfeed",
],
)
def test_template(self, tplname: str) -> None:
args = [f"--template={tplname}"] if tplname else []
spname = "test_spider"
spmodule = f"{self.project_name}.spiders.{spname}"
@ -35,15 +46,6 @@ class TestGenspiderCommand(TestCommandBase):
)
assert modify_time_after == modify_time_before
def test_template_basic(self):
self.test_template("basic")
def test_template_csvfeed(self):
self.test_template("csvfeed")
def test_template_xmlfeed(self):
self.test_template("xmlfeed")
def test_list(self):
assert self.call("genspider", "--list") == 0
@ -57,7 +59,8 @@ class TestGenspiderCommand(TestCommandBase):
self.proj_mod_path, "spiders", f"{self.project_name}.py"
).exists()
def test_same_filename_as_existing_spider(self, force=False):
@pytest.mark.parametrize("force", [True, False])
def test_same_filename_as_existing_spider(self, force: bool) -> None:
file_name = "example"
file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py")
assert self.call("genspider", file_name, "example.com") == 0
@ -90,83 +93,60 @@ class TestGenspiderCommand(TestCommandBase):
file_contents_after = file_path.read_text(encoding="utf-8")
assert file_contents_after == file_contents_before
def test_same_filename_as_existing_spider_force(self):
self.test_same_filename_as_existing_spider(force=True)
def test_url(self, url="test.com", domain="test.com"):
@pytest.mark.parametrize(
("url", "domain"),
[
("test.com", "test.com"),
("https://test.com", "test.com"),
],
)
def test_url(self, url: str, domain: str) -> None:
assert self.call("genspider", "--force", "test_name", url) == 0
assert (
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
).group(1)
== domain
m = self.find_in_file(
self.proj_mod_path / "spiders" / "test_name.py",
r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
)
assert (
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
).group(1)
== f"https://{domain}"
assert m is not None
assert m.group(1) == domain
m = self.find_in_file(
self.proj_mod_path / "spiders" / "test_name.py",
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
)
assert m is not None
assert m.group(1) == f"https://{domain}"
def test_url_schema(self):
self.test_url("https://test.com", "test.com")
def test_template_start_urls(
self, url="test.com", expected="https://test.com", template="basic"
):
@pytest.mark.parametrize(
("url", "expected", "template"),
[
# basic
("https://test.com", "https://test.com", "basic"),
("http://test.com", "http://test.com", "basic"),
("http://test.com/other/path", "http://test.com/other/path", "basic"),
("test.com/other/path", "https://test.com/other/path", "basic"),
# crawl
("https://test.com", "https://test.com", "crawl"),
("http://test.com", "http://test.com", "crawl"),
("http://test.com/other/path", "http://test.com/other/path", "crawl"),
("test.com/other/path", "https://test.com/other/path", "crawl"),
("test.com", "https://test.com", "crawl"),
# xmlfeed
("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"),
("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
# csvfeed
("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"),
("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
],
)
def test_template_start_urls(self, url: str, expected: str, template: str) -> None:
assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0
assert (
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
).group(1)
== expected
)
def test_genspider_basic_start_urls(self):
self.test_template_start_urls("https://test.com", "https://test.com", "basic")
self.test_template_start_urls("http://test.com", "http://test.com", "basic")
self.test_template_start_urls(
"http://test.com/other/path", "http://test.com/other/path", "basic"
)
self.test_template_start_urls(
"test.com/other/path", "https://test.com/other/path", "basic"
)
def test_genspider_crawl_start_urls(self):
self.test_template_start_urls("https://test.com", "https://test.com", "crawl")
self.test_template_start_urls("http://test.com", "http://test.com", "crawl")
self.test_template_start_urls(
"http://test.com/other/path", "http://test.com/other/path", "crawl"
)
self.test_template_start_urls(
"test.com/other/path", "https://test.com/other/path", "crawl"
)
self.test_template_start_urls("test.com", "https://test.com", "crawl")
def test_genspider_xmlfeed_start_urls(self):
self.test_template_start_urls(
"https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
)
self.test_template_start_urls(
"http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"
)
self.test_template_start_urls(
"test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
)
def test_genspider_csvfeed_start_urls(self):
self.test_template_start_urls(
"https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
)
self.test_template_start_urls(
"http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"
)
self.test_template_start_urls(
"test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
m = self.find_in_file(
self.proj_mod_path / "spiders" / "test_name.py",
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
)
assert m is not None
assert m.group(1) == expected
class TestGenspiderStandaloneCommand(TestProjectBase):
@ -174,7 +154,8 @@ class TestGenspiderStandaloneCommand(TestProjectBase):
self.call("genspider", "example", "example.com")
assert Path(self.temp_path, "example.py").exists()
def test_same_name_as_existing_file(self, force=False):
@pytest.mark.parametrize("force", [True, False])
def test_same_name_as_existing_file(self, force: bool) -> None:
file_name = "example"
file_path = Path(self.temp_path, file_name + ".py")
p, out, err = self.proc("genspider", file_name, "example.com")
@ -203,6 +184,3 @@ class TestGenspiderStandaloneCommand(TestProjectBase):
assert modify_time_after == modify_time_before
file_contents_after = file_path.read_text(encoding="utf-8")
assert file_contents_after == file_contents_before
def test_same_name_as_existing_file_force(self):
self.test_same_name_as_existing_file(force=True)

3
tests/test_command_runspider.py
View File

@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import inspect
import platform
import sys
@ -204,8 +205,6 @@ class MySpider(scrapy.Spider):
"TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",
],
)
import asyncio
if sys.platform != "win32":
loop = asyncio.new_event_loop()
else:

10
tests/test_commands.py
View File

@ -9,7 +9,7 @@ from io import StringIO
from pathlib import Path
from shutil import rmtree
from tempfile import TemporaryFile, mkdtemp
from typing import TYPE_CHECKING, Any
from typing import Any
from unittest import mock
import pytest
@ -22,9 +22,6 @@ from scrapy.utils.python import to_unicode
from scrapy.utils.reactor import _asyncio_reactor_path
from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
import os
class EmptyCommand(ScrapyCommand):
def short_desc(self) -> str:
@ -110,10 +107,11 @@ class TestProjectBase:
return p, to_unicode(stdout), to_unicode(stderr)
def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None:
@staticmethod
def find_in_file(filename: Path, regex: str) -> re.Match | None:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
with Path(filename).open("r", encoding="utf-8") as f:
with filename.open("r", encoding="utf-8") as f:
for line in f:
match = pattern.search(line)
if match is not None:

10
tests/test_core_downloader.py
View File

@ -4,7 +4,7 @@ import shutil
import warnings
from pathlib import Path
from tempfile import mkdtemp
from typing import Any, cast
from typing import TYPE_CHECKING, Any, cast
import OpenSSL.SSL
import pytest
@ -14,7 +14,6 @@ from twisted.trial import unittest
from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
from twisted.web.iweb import IBodyProducer
from scrapy.core.downloader import Slot
from scrapy.core.downloader.contextfactory import (
@ -29,6 +28,9 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
from tests.mockserver import PayloadResource, ssl_context_factory
if TYPE_CHECKING:
from twisted.web.iweb import IBodyProducer
class TestSlot:
def test_repr(self):
@ -78,12 +80,12 @@ class TestContextFactoryBase(unittest.TestCase):
agent = Agent(reactor, contextFactory=client_context_factory)
body_producer = _RequestBodyProducer(body.encode()) if body else None
response: TxResponse = cast(
TxResponse,
"TxResponse",
await maybe_deferred_to_future(
agent.request(
b"GET",
url.encode(),
bodyProducer=cast(IBodyProducer, body_producer),
bodyProducer=cast("IBodyProducer", body_producer),
)
),
)

9
tests/test_crawl.py
View File

@ -5,7 +5,7 @@ import logging
from ipaddress import IPv4Address
from socket import gethostbyname
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
from urllib.parse import urlencode, urlparse
import pytest
from testfixtures import LogCapture
@ -20,6 +20,7 @@ from scrapy.exceptions import CloseSpider, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import NON_EXISTING_RESOLVABLE
@ -255,8 +256,6 @@ class TestCrawl(TestCase):
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
from urllib.parse import urlencode
query = urlencode(
{
"raw": """\
@ -339,8 +338,6 @@ with multiples lines
@inlineCallbacks
def test_engine_status(self):
from scrapy.utils.engine import get_engine_status
est = []
def cb(response):
@ -357,8 +354,6 @@ with multiples lines
@inlineCallbacks
def test_format_engine_status(self):
from scrapy.utils.engine import format_engine_status
est = []
def cb(response):

10
tests/test_crawler.py
View File

@ -526,10 +526,10 @@ class TestCrawlerLogging:
crawler = get_crawler(MySpider)
assert get_scrapy_root_handler().level == logging.INFO
info_count = crawler.stats.get_value("log_count/INFO")
logging.debug("debug message")
logging.info("info message")
logging.warning("warning message")
logging.error("error message")
logging.debug("debug message") # noqa: LOG015
logging.info("info message") # noqa: LOG015
logging.warning("warning message") # noqa: LOG015
logging.error("error message") # noqa: LOG015
logged = log_file.read_text(encoding="utf-8")
@ -556,7 +556,7 @@ class TestCrawlerLogging:
configure_logging()
get_crawler(MySpider)
logging.debug("debug message")
logging.debug("debug message") # noqa: LOG015
logged = log_file.read_text(encoding="utf-8")

9
tests/test_downloader_handler_twisted_http2.py
View File

@ -15,10 +15,7 @@ from twisted.web.http import H2_ENABLED
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import (
deferred_f_from_coro_f,
maybe_deferred_to_future,
)
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver import ssl_context_factory
@ -46,7 +43,9 @@ class H2DownloadHandlerMixin:
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
# the import can fail when H2_ENABLED is False
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
from scrapy.core.downloader.handlers.http2 import ( # noqa: PLC0415
H2DownloadHandler,
)
return H2DownloadHandler

10
tests/test_downloader_handlers.py
View File

@ -13,7 +13,7 @@ from unittest import mock
import pytest
from twisted.cred import checkers, credentials, portal
from twisted.internet.defer import inlineCallbacks
from twisted.protocols.ftp import FTPFactory, FTPRealm
from twisted.protocols.ftp import ConnectionLost, FTPFactory, FTPRealm
from twisted.trial import unittest
from w3lib.url import path_to_file_uri
@ -27,10 +27,7 @@ from scrapy.http import HtmlResponse, Request, Response
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.defer import (
deferred_f_from_coro_f,
maybe_deferred_to_future,
)
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
@ -184,7 +181,7 @@ class TestS3:
@contextlib.contextmanager
def _mocked_date(self, date):
try:
import botocore.auth # noqa: F401
import botocore.auth # noqa: F401,PLC0415
except ImportError:
yield
else:
@ -443,7 +440,6 @@ class TestFTP(TestFTPBase):
pytest.skip(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
)
from twisted.protocols.ftp import ConnectionLost
meta = dict(self.req_meta)
meta.update({"ftp_password": "invalid"})

128
tests/test_downloadermiddleware_httpcompression.py
View File

@ -50,6 +50,23 @@ FORMAT = {
}
def _skip_if_no_br() -> None:
try:
try:
import brotli # noqa: F401,PLC0415
except ImportError:
import brotlicffi # noqa: F401,PLC0415
except ImportError:
pytest.skip("no brotli support")
def _skip_if_no_zstd() -> None:
try:
import zstandard # noqa: F401,PLC0415
except ImportError:
pytest.skip("no zstd support (zstandard)")
class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
@ -123,13 +140,8 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
response = self._getresponse("br")
request = response.request
assert response.headers["Content-Encoding"] == b"br"
@ -143,11 +155,11 @@ class TestHttpCompression:
def test_process_response_br_unsupported(self):
try:
try:
import brotli # noqa: F401
import brotli # noqa: F401,PLC0415
pytest.skip("Requires not having brotli support")
except ImportError:
import brotlicffi # noqa: F401
import brotlicffi # noqa: F401,PLC0415
pytest.skip("Requires not having brotli support")
except ImportError:
@ -176,10 +188,8 @@ class TestHttpCompression:
assert newresponse.headers.getlist("Content-Encoding") == [b"br"]
def test_process_response_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
raw_content = None
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
@ -198,7 +208,7 @@ class TestHttpCompression:
def test_process_response_zstd_unsupported(self):
try:
import zstandard # noqa: F401
import zstandard # noqa: F401,PLC0415
pytest.skip("Requires not having zstandard support")
except ImportError:
@ -513,13 +523,8 @@ class TestHttpCompression:
mw.process_response(response.request, response, spider)
def test_compression_bomb_setting_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
self._test_compression_bomb_setting("br")
def test_compression_bomb_setting_deflate(self):
@ -529,10 +534,8 @@ class TestHttpCompression:
self._test_compression_bomb_setting("gzip")
def test_compression_bomb_setting_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
self._test_compression_bomb_setting("zstd")
def _test_compression_bomb_spider_attr(self, compression_id):
@ -549,13 +552,8 @@ class TestHttpCompression:
mw.process_response(response.request, response, spider)
def test_compression_bomb_spider_attr_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
self._test_compression_bomb_spider_attr("br")
def test_compression_bomb_spider_attr_deflate(self):
@ -565,10 +563,8 @@ class TestHttpCompression:
self._test_compression_bomb_spider_attr("gzip")
def test_compression_bomb_spider_attr_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
@ -583,13 +579,8 @@ class TestHttpCompression:
mw.process_response(response.request, response, spider)
def test_compression_bomb_request_meta_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
self._test_compression_bomb_request_meta("br")
def test_compression_bomb_request_meta_deflate(self):
@ -599,10 +590,8 @@ class TestHttpCompression:
self._test_compression_bomb_request_meta("gzip")
def test_compression_bomb_request_meta_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
self._test_compression_bomb_request_meta("zstd")
def _test_download_warnsize_setting(self, compression_id):
@ -632,13 +621,8 @@ class TestHttpCompression:
)
def test_download_warnsize_setting_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
self._test_download_warnsize_setting("br")
def test_download_warnsize_setting_deflate(self):
@ -648,10 +632,8 @@ class TestHttpCompression:
self._test_download_warnsize_setting("gzip")
def test_download_warnsize_setting_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
self._test_download_warnsize_setting("zstd")
def _test_download_warnsize_spider_attr(self, compression_id):
@ -683,13 +665,8 @@ class TestHttpCompression:
)
def test_download_warnsize_spider_attr_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
self._test_download_warnsize_spider_attr("br")
def test_download_warnsize_spider_attr_deflate(self):
@ -699,10 +676,8 @@ class TestHttpCompression:
self._test_download_warnsize_spider_attr("gzip")
def test_download_warnsize_spider_attr_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
self._test_download_warnsize_spider_attr("zstd")
def _test_download_warnsize_request_meta(self, compression_id):
@ -732,13 +707,8 @@ class TestHttpCompression:
)
def test_download_warnsize_request_meta_br(self):
try:
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pytest.skip("no brotli")
_skip_if_no_br()
self._test_download_warnsize_request_meta("br")
def test_download_warnsize_request_meta_deflate(self):
@ -748,8 +718,6 @@ class TestHttpCompression:
self._test_download_warnsize_request_meta("gzip")
def test_download_warnsize_request_meta_zstd(self):
try:
import zstandard # noqa: F401
except ImportError:
pytest.skip("no zstd support (zstandard)")
_skip_if_no_zstd()
self._test_download_warnsize_request_meta("zstd")

26
tests/test_feedexport.py
View File

@ -5,7 +5,9 @@ import csv
import gzip
import json
import lzma
import marshal
import os
import pickle
import random
import shutil
import string
@ -25,6 +27,7 @@ from urllib.request import pathname2url
import lxml.etree
import pytest
from packaging.version import Version
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
@ -79,7 +82,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.
"""
from google.cloud.storage import Blob, Bucket, Client
from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415
client_mock = mock.create_autospec(Client)
@ -507,7 +510,7 @@ class TestS3FeedStorage(unittest.TestCase):
class TestGCSFeedStorage(unittest.TestCase):
def test_parse_settings(self):
try:
from google.cloud.storage import Client # noqa: F401
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
@ -521,7 +524,7 @@ class TestGCSFeedStorage(unittest.TestCase):
def test_parse_empty_acl(self):
try:
from google.cloud.storage import Client # noqa: F401
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
@ -538,7 +541,7 @@ class TestGCSFeedStorage(unittest.TestCase):
@deferred_f_from_coro_f
async def test_store(self):
try:
from google.cloud.storage import Client # noqa: F401
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
@ -976,7 +979,6 @@ class TestFeedExport(TestFeedExportBase):
)
data = await self.exported_data(items, settings)
expected = [{k: v for k, v in row.items() if v} for row in rows]
import pickle
result = self._load_until_eof(data["pickle"], load_func=pickle.load)
assert result == expected
@ -997,7 +999,6 @@ class TestFeedExport(TestFeedExportBase):
)
data = await self.exported_data(items, settings)
expected = [{k: v for k, v in row.items() if v} for row in rows]
import marshal
result = self._load_until_eof(data["marshal"], load_func=marshal.load)
assert result == expected
@ -2300,9 +2301,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
@deferred_f_from_coro_f
async def test_exports_compatibility_with_postproc(self):
import marshal
import pickle
filename_to_expected = {
self._named_tempfile("csv"): b"foo\r\nbar\r\n",
self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',
@ -2484,7 +2482,6 @@ class TestBatchDeliveries(TestFeedExportBase):
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
import pickle
for batch in data["pickle"]:
got_batch = self._load_until_eof(batch, load_func=pickle.load)
@ -2505,7 +2502,6 @@ class TestBatchDeliveries(TestFeedExportBase):
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
import marshal
for batch in data["marshal"]:
got_batch = self._load_until_eof(batch, load_func=marshal.load)
@ -2714,9 +2710,8 @@ class TestBatchDeliveries(TestFeedExportBase):
stubs = []
def open(self, *args, **kwargs):
from botocore import __version__ as botocore_version
from botocore.stub import ANY, Stubber
from packaging.version import Version
from botocore import __version__ as botocore_version # noqa: PLC0415
from botocore.stub import ANY, Stubber # noqa: PLC0415
expected_params = {
"Body": ANY,
@ -2912,10 +2907,9 @@ class TestURIParams(ABC):
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
else:
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
feed_exporter = crawler.get_extension(FeedExporter)
return crawler, feed_exporter
def test_default(self):

10
tests/test_http2_client_protocol.py
View File

@ -243,7 +243,7 @@ class TestHttps2ClientProtocol(TestCase):
self.conn_closed_deferred = Deferred()
from scrapy.core.http2.protocol import H2ClientFactory
from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415
h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
client_endpoint = SSL4ClientEndpoint(
@ -445,7 +445,7 @@ class TestHttps2ClientProtocol(TestCase):
def test_received_dataloss_response(self):
"""In case when value of Header Content-Length != len(Received Data)
ProtocolError is raised"""
from h2.exceptions import InvalidBodyLengthError
from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415
request = Request(url=self.get_url("/dataloss"))
with pytest.raises(ResponseFailed) as exc_info:
@ -525,7 +525,7 @@ class TestHttps2ClientProtocol(TestCase):
def assert_inactive_stream(failure):
assert failure.check(ResponseFailed) is not None
from scrapy.core.http2.stream import InactiveStreamClosed
from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415
assert any(
isinstance(e, InactiveStreamClosed) for e in failure.value.reasons
@ -593,7 +593,7 @@ class TestHttps2ClientProtocol(TestCase):
assert str(response.ip_address) == "127.0.0.1"
async def _check_invalid_netloc(self, url: str) -> None:
from scrapy.core.http2.stream import InvalidHostname
from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415
request = Request(url)
with pytest.raises(InvalidHostname) as exc_info:
@ -630,7 +630,7 @@ class TestHttps2ClientProtocol(TestCase):
yield self.make_request_dfd(request)
for err in exc_info.value.reasons:
from scrapy.core.http2.protocol import H2ClientProtocol
from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415
if isinstance(err, TimeoutError):
assert (

2
tests/test_http_request.py
View File

@ -1441,7 +1441,7 @@ class TestXmlRpcRequest(TestRequest):
)
assert r.method == "POST"
assert r.encoding == kwargs.get("encoding", "utf-8")
assert r.dont_filter, True
assert r.dont_filter
def test_xmlrpc_dumps(self):
self._test_request(params=("value",))

2
tests/test_logformatter.py
View File

@ -110,7 +110,7 @@ class TestLogFormatter:
assert logkws["level"] == unsupported_value
with pytest.raises(TypeError):
logging.log(logkws["level"], "message")
logging.log(logkws["level"], "message") # noqa: LOG015
def test_dropitem_custom_log_level(self):
item = {}

15
tests/test_pipeline_files.py
View File

@ -5,6 +5,7 @@ import time
import warnings
from abc import ABC, abstractmethod
from datetime import datetime
from ftplib import FTP
from io import BytesIO
from pathlib import Path
from posixpath import split
@ -29,9 +30,7 @@ from scrapy.pipelines.files import (
GCSFilesStore,
S3FilesStore,
)
from scrapy.utils.test import (
get_crawler,
)
from scrapy.utils.test import get_crawler
from tests.mockserver import MockFTPServer
from .test_pipeline_media import _mocked_download_func
@ -40,7 +39,7 @@ from .test_pipeline_media import _mocked_download_func
def get_gcs_content_and_delete(
bucket: Any, path: str
) -> tuple[bytes, list[dict[str, str]], Any]:
from google.cloud import storage
from google.cloud import storage # noqa: PLC0415
client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
bucket = client.get_bucket(bucket)
@ -59,8 +58,6 @@ def get_ftp_content_and_delete(
password: str,
use_active_mode: bool = False,
) -> bytes:
from ftplib import FTP
ftp = FTP()
ftp.connect(host, port)
ftp.login(username, password)
@ -553,7 +550,7 @@ class TestS3FilesStore(unittest.TestCase):
content_type = "image/png"
store = S3FilesStore(uri)
from botocore.stub import Stubber
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
stub.add_response(
@ -591,7 +588,7 @@ class TestS3FilesStore(unittest.TestCase):
last_modified = datetime(2019, 12, 1)
store = S3FilesStore(uri)
from botocore.stub import Stubber
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
stub.add_response(
@ -650,7 +647,7 @@ class TestGCSFilesStore(unittest.TestCase):
already uploaded files.
"""
try:
import google.cloud.storage # noqa: F401
import google.cloud.storage # noqa: F401,PLC0415
except ModuleNotFoundError:
pytest.skip("google-cloud-storage is not installed")
with (

2
tests/test_proxy_connect.py
View File

@ -74,7 +74,7 @@ class TestProxyConnect(TestCase):
def setUp(self):
try:
import mitmproxy # noqa: F401
import mitmproxy # noqa: F401,PLC0415
except ImportError:
pytest.skip("mitmproxy is not installed")

27
tests/test_robotstxt_interface.py
View File

@ -1,21 +1,19 @@
import pytest
from scrapy.robotstxt import decode_robotstxt
from scrapy.robotstxt import (
ProtegoRobotParser,
PythonRobotParser,
RerpRobotParser,
decode_robotstxt,
)
def rerp_available():
# check if robotexclusionrulesparser is installed
try:
from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401
except ImportError:
return False
return True
def protego_available():
# check if protego parser is installed
try:
from protego import Protego # noqa: F401
from robotexclusionrulesparser import ( # noqa: PLC0415
RobotExclusionRulesParser, # noqa: F401
)
except ImportError:
return False
return True
@ -134,8 +132,6 @@ class TestDecodeRobotsTxt:
class TestPythonRobotParser(BaseRobotParserTest):
def setup_method(self):
from scrapy.robotstxt import PythonRobotParser
super()._setUp(PythonRobotParser)
def test_length_based_precedence(self):
@ -150,19 +146,14 @@ class TestPythonRobotParser(BaseRobotParserTest):
@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
class TestRerpRobotParser(BaseRobotParserTest):
def setup_method(self):
from scrapy.robotstxt import RerpRobotParser
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
pytest.skip("Rerp does not support length based directives precedence.")
@pytest.mark.skipif(not protego_available(), reason="Protego parser is not installed")
class TestProtegoRobotParser(BaseRobotParserTest):
def setup_method(self):
from scrapy.robotstxt import ProtegoRobotParser
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):

6
tests/test_scrapy__getattr__.py
View File

@ -3,7 +3,9 @@ import warnings
def test_deprecated_twisted_version():
with warnings.catch_warnings(record=True) as warns:
from scrapy import twisted_version # pylint: disable=no-name-in-module
from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module
twisted_version,
)
assert twisted_version is not None
assert isinstance(twisted_version, tuple)
@ -15,7 +17,7 @@ def test_deprecated_twisted_version():
def test_deprecated_concurrent_requests_per_ip_attribute():
with warnings.catch_warnings(record=True) as warns:
from scrapy.settings.default_settings import (
from scrapy.settings.default_settings import ( # noqa: PLC0415
CONCURRENT_REQUESTS_PER_IP,
)

7
tests/test_settings/__init__.py
View File

@ -6,6 +6,7 @@ from unittest import mock
import pytest
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.settings import (
SETTINGS_PRIORITIES,
BaseSettings,
@ -13,6 +14,8 @@ from scrapy.settings import (
SettingsAttribute,
get_settings_priority,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from . import default_settings
@ -447,10 +450,6 @@ class TestSettings:
assert mydict["key"] == "val"
def test_passing_objects_as_values(self):
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
class TestPipeline:
def process_item(self, i, s):
return i

8
tests/test_spider.py
View File

@ -1,7 +1,9 @@
from __future__ import annotations
import gzip
import re
import warnings
from datetime import datetime
from io import BytesIO
from logging import ERROR, WARNING
from pathlib import Path
@ -297,8 +299,6 @@ class TestCrawlSpider(TestSpider):
)
class _CrawlSpider(self.spider_class):
import re
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@ -635,8 +635,6 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
from datetime import datetime
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year > 2008:
@ -706,8 +704,6 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
from datetime import datetime
for entry in entries:
date_time = datetime.strptime(
entry["lastmod"].split("T")[0], "%Y-%m-%d"

9
tests/test_spidermiddleware_referer.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
from typing import Any, cast
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlparse
import pytest
@ -33,6 +33,9 @@ from scrapy.spidermiddlewares.referer import (
)
from scrapy.spiders import Spider
if TYPE_CHECKING:
from collections.abc import Sequence
@pytest.fixture
def spider() -> Spider:
@ -971,7 +974,7 @@ class TestPolicyHeaderPrecedence004(
class TestReferrerOnRedirect(TestRefererMiddleware):
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
scenarii: list[
scenarii: Sequence[
tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None]
] = [ # type: ignore[assignment]
(
@ -1041,7 +1044,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
request.url, headers={"Location": url}, status=status
)
request = cast(
Request, redirectmw.process_response(request, response, spider)
"Request", redirectmw.process_response(request, response, spider)
)
referrermw.request_scheduled(request, spider)

8
tests/test_toplevel.py
View File

@ -10,26 +10,26 @@ def test_version_info():
def test_request_shortcut():
from scrapy.http import FormRequest, Request
from scrapy.http import FormRequest, Request # noqa: PLC0415
assert scrapy.Request is Request
assert scrapy.FormRequest is FormRequest
def test_spider_shortcut():
from scrapy.spiders import Spider
from scrapy.spiders import Spider # noqa: PLC0415
assert scrapy.Spider is Spider
def test_selector_shortcut():
from scrapy.selector import Selector
from scrapy.selector import Selector # noqa: PLC0415
assert scrapy.Selector is Selector
def test_item_shortcut():
from scrapy.item import Field, Item
from scrapy.item import Field, Item # noqa: PLC0415
assert scrapy.Item is Item
assert scrapy.Field is Field

3
tests/test_utils_display.py
View File

@ -1,3 +1,4 @@
import builtins
from io import StringIO
from unittest import mock
@ -74,8 +75,6 @@ def test_pformat_windows(isatty, version, terminal_processing):
def test_pformat_no_pygments(isatty):
isatty.return_value = True
import builtins
real_import = builtins.__import__
def mock_import(name, globals, locals, fromlist, level):

2
tests/test_utils_spider.py
View File

@ -24,7 +24,7 @@ def test_iterate_spider_output():
def test_iter_spider_classes():
import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
import tests.test_utils_spider # noqa: PLW0406,PLC0415 # pylint: disable=import-self
it = iter_spider_classes(tests.test_utils_spider)
assert set(it) == {MySpider1, MySpider2}

3
tests/test_utils_url.py
View File

@ -1,4 +1,5 @@
import warnings
from importlib import import_module
import pytest
@ -461,8 +462,6 @@ def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
obj_type = "attribute" if obj_name == "_safe_chars" else "function"
message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
from importlib import import_module
getattr(import_module("scrapy.utils.url"), obj_name)
assert isinstance(warns[0].message, Warning)

7
tests/test_webclient.py
View File

@ -18,11 +18,10 @@ from twisted.internet.testing import StringTransport
from twisted.protocols.policies import WrappingFactory
from twisted.trial import unittest
from twisted.web import resource, server, static, util
from twisted.web.client import _makeGetterFactory
from scrapy.core.downloader import webclient as client
from scrapy.core.downloader.contextfactory import (
ScrapyClientContextFactory,
)
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.http import Headers, Request
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes, to_unicode
@ -51,8 +50,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
f.deferred.addCallback(response_transform or (lambda r: r.body))
return f
from twisted.web.client import _makeGetterFactory
return _makeGetterFactory(
to_bytes(url),
_clientfactory,

22
tox.ini
View File

@ -46,17 +46,21 @@ install_command =
[testenv:typing]
basepython = python3.9
deps =
mypy==1.14.0
typing-extensions==4.12.2
types-lxml==2024.12.13
types-Pygments==2.18.0.20240506
botocore-stubs==1.35.90
boto3-stubs[s3]==1.35.90
mypy==1.16.1
typing-extensions==4.14.1
types-defusedxml==0.7.0.20250516
types-lxml==2025.3.30
types-pexpect==4.9.0.20250516
types-Pygments==2.19.0.20250516
botocore-stubs==1.38.46
boto3-stubs[s3]==1.39.3
itemadapter==0.11.0
Protego==0.5.0
w3lib==2.3.1
attrs >= 18.2.0
Pillow >= 10.3.0
pyOpenSSL >= 24.2.1
pytest >= 8.2.0
w3lib >= 2.2.0
commands =
mypy {posargs:scrapy tests}
@ -80,14 +84,14 @@ commands =
basepython = python3
deps =
{[testenv:extra-deps]deps}
pylint==3.3.3
pylint==3.3.7
commands =
pylint conftest.py docs extras scrapy tests
[testenv:twinecheck]
basepython = python3
deps =
twine==6.0.1
twine==6.1.0
build==1.2.2.post1
commands =
python -m build --sdist