mirror of https://github.com/scrapy/scrapy.git
Merge pull request #6581 from wRAR/ruff-rules-3
Ruff: enable other useful rules, turn on autofixing
This commit is contained in:
Andrey Rakhmatullin
GitHub
commit
57a5460529
|
|
@ -3,6 +3,7 @@ repos:
|
|||
rev: v0.8.1
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [ --fix ]
|
||||
- repo: https://github.com/psf/black.git
|
||||
rev: 24.4.2
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -240,10 +240,20 @@ extend-select = [
|
|||
"ISC",
|
||||
# flake8-logging
|
||||
"LOG",
|
||||
# Perflint
|
||||
"PERF",
|
||||
# pygrep-hooks
|
||||
"PGH",
|
||||
# flake8-pie
|
||||
"PIE",
|
||||
# flake8-pyi
|
||||
"PYI",
|
||||
# flake8-quotes
|
||||
"Q",
|
||||
# flake8-return
|
||||
"RET",
|
||||
# flake8-raise
|
||||
"RSE",
|
||||
# flake8-bandit
|
||||
"S",
|
||||
# flake8-slots
|
||||
|
|
@ -254,6 +264,8 @@ extend-select = [
|
|||
"TC",
|
||||
# pyupgrade
|
||||
"UP",
|
||||
# pycodestyle warnings
|
||||
"W",
|
||||
# flake8-2020
|
||||
"YTT",
|
||||
]
|
||||
|
|
@ -306,6 +318,8 @@ ignore = [
|
|||
"D402",
|
||||
# First word of the first line should be properly capitalized
|
||||
"D403",
|
||||
# `try`-`except` within a loop incurs performance overhead
|
||||
"PERF203",
|
||||
# Use of `assert` detected; needed for mypy
|
||||
"S101",
|
||||
# FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
if len(args) < 1:
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
if len(args) > 1:
|
||||
raise UsageError(
|
||||
"running 'scrapy crawl' with more than one spider is not supported"
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
if len(args) != 1:
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
|
||||
editor = self.settings["EDITOR"]
|
||||
assert self.crawler_process
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args: list[str], opts: Namespace) -> None:
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
request = Request(
|
||||
args[0],
|
||||
callback=self._print_response,
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ class Command(ScrapyCommand):
|
|||
print(template_file.read_text(encoding="utf-8"))
|
||||
return
|
||||
if len(args) != 2:
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
|
||||
name, url = args[0:2]
|
||||
url = verify_url_scheme(url)
|
||||
|
|
|
|||
|
|
@ -225,8 +225,9 @@ class Command(BaseRunSpiderCommand):
|
|||
cb_kwargs: dict[str, Any] | None = None,
|
||||
) -> Deferred[Any]:
|
||||
cb_kwargs = cb_kwargs or {}
|
||||
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
|
||||
return d
|
||||
return maybeDeferred(
|
||||
self.iterate_spider_output, callback(response, **cb_kwargs)
|
||||
)
|
||||
|
||||
def get_callback_from_rules(
|
||||
self, spider: Spider, response: Response
|
||||
|
|
@ -398,7 +399,7 @@ class Command(BaseRunSpiderCommand):
|
|||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
# parse arguments
|
||||
if not len(args) == 1 or not is_url(args[0]):
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
url = args[0]
|
||||
|
||||
# prepare spidercls
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
if len(args) != 1:
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
filename = Path(args[0])
|
||||
if not filename.exists():
|
||||
raise UsageError(f"File not found: {filename}\n")
|
||||
|
|
|
|||
|
|
@ -61,7 +61,6 @@ class Command(ScrapyCommand):
|
|||
"""You can use this function to update the Scrapy objects that will be
|
||||
available in the shell
|
||||
"""
|
||||
pass
|
||||
|
||||
def run(self, args: list[str], opts: Namespace) -> None:
|
||||
url = args[0] if args else None
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
if len(args) not in (1, 2):
|
||||
raise UsageError()
|
||||
raise UsageError
|
||||
|
||||
project_name = args[0]
|
||||
|
||||
|
|
|
|||
|
|
@ -81,7 +81,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
|
|||
:param spider: the spider object for the current crawl
|
||||
:type spider: :class:`~scrapy.spiders.Spider`
|
||||
"""
|
||||
pass
|
||||
|
||||
def close(self, reason: str) -> Deferred[None] | None:
|
||||
"""
|
||||
|
|
@ -91,14 +90,13 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
|
|||
:param reason: a string which describes the reason why the spider was closed
|
||||
:type reason: :class:`str`
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def has_pending_requests(self) -> bool:
|
||||
"""
|
||||
``True`` if the scheduler has enqueued requests, ``False`` otherwise
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def enqueue_request(self, request: Request) -> bool:
|
||||
|
|
@ -112,7 +110,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
|
|||
For reference, the default Scrapy scheduler returns ``False`` when the
|
||||
request is rejected by the dupefilter.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def next_request(self) -> Request | None:
|
||||
|
|
@ -124,7 +122,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
|
|||
to the downloader in the current reactor cycle. The engine will continue
|
||||
calling ``next_request`` until ``has_pending_requests`` is ``False``.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Scheduler(BaseScheduler):
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class OffsiteMiddleware:
|
|||
|
||||
def process_request(self, request: Request, spider: Spider) -> None:
|
||||
if request.dont_filter or self.should_follow(request, spider):
|
||||
return None
|
||||
return
|
||||
domain = urlparse_cached(request).hostname
|
||||
if domain and domain not in self.domains_seen:
|
||||
self.domains_seen.add(domain)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from twisted.internet.defer import Deferred, maybeDeferred
|
||||
|
||||
|
|
@ -31,8 +31,6 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class RobotsTxtMiddleware:
|
||||
DOWNLOAD_PRIORITY: int = 1000
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ class BaseDupeFilter:
|
|||
|
||||
def log(self, request: Request, spider: Spider) -> None:
|
||||
"""Log that a request has been filtered"""
|
||||
pass
|
||||
|
||||
|
||||
class RFPDupeFilter(BaseDupeFilter):
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ from typing import Any
|
|||
class NotConfigured(Exception):
|
||||
"""Indicates a missing configuration situation"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class _InvalidOutput(TypeError):
|
||||
"""
|
||||
|
|
@ -22,8 +20,6 @@ class _InvalidOutput(TypeError):
|
|||
Internal and undocumented, it should not be raised or caught by user code.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
# HTTP and crawling
|
||||
|
||||
|
|
@ -35,8 +31,6 @@ class IgnoreRequest(Exception):
|
|||
class DontCloseSpider(Exception):
|
||||
"""Request the spider not to be closed yet"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class CloseSpider(Exception):
|
||||
"""Raise this from callbacks to request the spider to be closed"""
|
||||
|
|
@ -64,14 +58,10 @@ class StopDownload(Exception):
|
|||
class DropItem(Exception):
|
||||
"""Drop item from the item pipeline"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NotSupported(Exception):
|
||||
"""Indicates a feature or method is not supported"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
# Commands
|
||||
|
||||
|
|
@ -89,10 +79,6 @@ class ScrapyDeprecationWarning(Warning):
|
|||
DeprecationWarning is silenced on Python 2.7+
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ContractFail(AssertionError):
|
||||
"""Error raised in case of a failing contract"""
|
||||
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -586,7 +586,7 @@ class FeedExporter:
|
|||
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
|
||||
"""
|
||||
storage = self._get_storage(uri, feed_options)
|
||||
slot = FeedSlot(
|
||||
return FeedSlot(
|
||||
storage=storage,
|
||||
uri=uri,
|
||||
format=feed_options["format"],
|
||||
|
|
@ -600,7 +600,6 @@ class FeedExporter:
|
|||
settings=self.settings,
|
||||
crawler=self.crawler,
|
||||
)
|
||||
return slot
|
||||
|
||||
def item_scraped(self, item: Any, spider: Spider) -> None:
|
||||
slots = []
|
||||
|
|
|
|||
|
|
@ -282,8 +282,7 @@ class DbmCacheStorage:
|
|||
headers = Headers(data["headers"])
|
||||
body = data["body"]
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
return respcls(url=url, headers=headers, status=status, body=body)
|
||||
|
||||
def store_response(
|
||||
self, spider: Spider, request: Request, response: Response
|
||||
|
|
@ -349,8 +348,7 @@ class FilesystemCacheStorage:
|
|||
status = metadata["status"]
|
||||
headers = Headers(headers_raw_to_dict(rawheaders))
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
return respcls(url=url, headers=headers, status=status, body=body)
|
||||
|
||||
def store_response(
|
||||
self, spider: Spider, request: Request, response: Response
|
||||
|
|
|
|||
|
|
@ -157,8 +157,7 @@ class PostProcessingManager(IOBase):
|
|||
return True
|
||||
|
||||
def _load_plugins(self, plugins: list[Any]) -> list[Any]:
|
||||
plugins = [load_object(plugin) for plugin in plugins]
|
||||
return plugins
|
||||
return [load_object(plugin) for plugin in plugins]
|
||||
|
||||
def _get_head_plugin(self) -> Any:
|
||||
prev = self.file
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ For actual link extractors implementation see scrapy.linkextractors, or
|
|||
its documentation in: docs/topics/link-extractors.rst
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Link:
|
||||
"""Link objects represent an extracted link by the LinkExtractor.
|
||||
|
|
@ -39,7 +37,7 @@ class Link:
|
|||
self.fragment: str = fragment
|
||||
self.nofollow: bool = nofollow
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, Link):
|
||||
raise NotImplementedError
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -253,8 +253,7 @@ class LxmlLinkExtractor:
|
|||
if self.canonicalize:
|
||||
for link in links:
|
||||
link.url = canonicalize_url(link.url)
|
||||
links = self.link_extractor._process_links(links)
|
||||
return links
|
||||
return self.link_extractor._process_links(links)
|
||||
|
||||
def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]:
|
||||
return self.link_extractor._extract_links(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -5,16 +5,7 @@ import logging
|
|||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Literal,
|
||||
NoReturn,
|
||||
TypedDict,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Any, Literal, NoReturn, TypedDict, Union, cast
|
||||
|
||||
from twisted import version as twisted_version
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
|
|
@ -41,8 +32,6 @@ if TYPE_CHECKING:
|
|||
from scrapy.http import Response
|
||||
from scrapy.utils.request import RequestFingerprinter
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class FileInfo(TypedDict):
|
||||
url: str
|
||||
|
|
@ -293,12 +282,12 @@ class MediaPipeline(ABC):
|
|||
self, request: Request, info: SpiderInfo, *, item: Any = None
|
||||
) -> Deferred[FileInfo | None]:
|
||||
"""Check request before starting download"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]:
|
||||
"""Returns the media requests to download"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def media_downloaded(
|
||||
|
|
@ -310,14 +299,14 @@ class MediaPipeline(ABC):
|
|||
item: Any = None,
|
||||
) -> FileInfo:
|
||||
"""Handler for success downloads"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def media_failed(
|
||||
self, failure: Failure, request: Request, info: SpiderInfo
|
||||
) -> NoReturn:
|
||||
"""Handler for failed downloads"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
def item_completed(
|
||||
self, results: list[FileInfoOrError], item: Any, info: SpiderInfo
|
||||
|
|
@ -345,4 +334,4 @@ class MediaPipeline(ABC):
|
|||
item: Any = None,
|
||||
) -> str:
|
||||
"""Returns the path where downloaded media should be stored"""
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ class HostResolution:
|
|||
self.name: str = name
|
||||
|
||||
def cancel(self) -> None:
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@provider(IResolutionReceiver)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ class RobotParser(metaclass=ABCMeta):
|
|||
:param robotstxt_body: content of a robots.txt_ file.
|
||||
:type robotstxt_body: bytes
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
|
||||
|
|
@ -64,7 +63,6 @@ class RobotParser(metaclass=ABCMeta):
|
|||
:param user_agent: User agent
|
||||
:type user_agent: str or bytes
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class PythonRobotParser(RobotParser):
|
||||
|
|
@ -79,8 +77,7 @@ class PythonRobotParser(RobotParser):
|
|||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
|
||||
spider = None if not crawler else crawler.spider
|
||||
o = cls(robotstxt_body, spider)
|
||||
return o
|
||||
return cls(robotstxt_body, spider)
|
||||
|
||||
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
|
||||
user_agent = to_unicode(user_agent)
|
||||
|
|
@ -100,8 +97,7 @@ class RerpRobotParser(RobotParser):
|
|||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
|
||||
spider = None if not crawler else crawler.spider
|
||||
o = cls(robotstxt_body, spider)
|
||||
return o
|
||||
return cls(robotstxt_body, spider)
|
||||
|
||||
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
|
||||
user_agent = to_unicode(user_agent)
|
||||
|
|
@ -120,8 +116,7 @@ class ProtegoRobotParser(RobotParser):
|
|||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
|
||||
spider = None if not crawler else crawler.spider
|
||||
o = cls(robotstxt_body, spider)
|
||||
return o
|
||||
return cls(robotstxt_body, spider)
|
||||
|
||||
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
|
||||
user_agent = to_unicode(user_agent)
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class ReferrerPolicy:
|
|||
name: str
|
||||
|
||||
def referrer(self, response_url: str, request_url: str) -> str | None:
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
def stripped_referrer(self, url: str) -> str | None:
|
||||
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
|
||||
|
|
|
|||
|
|
@ -7,10 +7,7 @@ _T = TypeVar("_T")
|
|||
|
||||
|
||||
async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]:
|
||||
results = []
|
||||
async for x in result:
|
||||
results.append(x)
|
||||
return results
|
||||
return [x async for x in result]
|
||||
|
||||
|
||||
async def as_async_generator(
|
||||
|
|
|
|||
|
|
@ -235,8 +235,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str
|
|||
continue
|
||||
args.append(name)
|
||||
else:
|
||||
for name in sig.parameters.keys():
|
||||
args.append(name)
|
||||
args = list(sig.parameters)
|
||||
|
||||
if stripself and args and args[0] == "self":
|
||||
args = args[1:]
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
|
|||
return reactor.listenTCP(0, factory, interface=host)
|
||||
if len(portrange) == 1:
|
||||
return reactor.listenTCP(portrange[0], factory, interface=host)
|
||||
for x in range(portrange[0], portrange[1] + 1):
|
||||
for x in range(portrange[0], portrange[1] + 1): # noqa: RET503
|
||||
try:
|
||||
return reactor.listenTCP(x, factory, interface=host)
|
||||
except error.CannotListenError:
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ def get_meta_refresh(
|
|||
return _metaref_cache[response]
|
||||
|
||||
|
||||
def response_status_message(status: bytes | float | int | str) -> str:
|
||||
def response_status_message(status: bytes | float | str) -> str:
|
||||
"""Return status code plus status text descriptive message"""
|
||||
status_int = int(status)
|
||||
message = http.RESPONSES.get(status_int, "Unknown Status")
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
|
|||
status = await get_from_asyncio_queue(response.status)
|
||||
self.logger.info(f"Got response {status}, req_id {req_id}")
|
||||
if req_id > 0:
|
||||
return
|
||||
return None
|
||||
reqs = []
|
||||
for i in range(1, 3):
|
||||
req = Request(self.start_urls[0], dont_filter=True, meta={"req_id": i})
|
||||
|
|
@ -393,8 +393,8 @@ class DuplicateStartRequestsSpider(MockServerSpider):
|
|||
dupe_factor = 3
|
||||
|
||||
def start_requests(self):
|
||||
for i in range(0, self.distinct_urls):
|
||||
for j in range(0, self.dupe_factor):
|
||||
for i in range(self.distinct_urls):
|
||||
for j in range(self.dupe_factor):
|
||||
url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
|
||||
yield Request(url, dont_filter=self.dont_filter)
|
||||
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class AddonManagerTest(unittest.TestCase):
|
|||
def test_notconfigured(self):
|
||||
class NotConfiguredAddon:
|
||||
def update_settings(self, settings):
|
||||
raise NotConfigured()
|
||||
raise NotConfigured
|
||||
|
||||
settings_dict = {
|
||||
"ADDONS": {NotConfiguredAddon: 0},
|
||||
|
|
|
|||
|
|
@ -178,27 +178,23 @@ class TestSpider(Spider):
|
|||
"""method with no url
|
||||
@returns items 1 1
|
||||
"""
|
||||
pass
|
||||
|
||||
def custom_form(self, response):
|
||||
"""
|
||||
@url http://scrapy.org
|
||||
@custom_form
|
||||
"""
|
||||
pass
|
||||
|
||||
def invalid_regex(self, response):
|
||||
"""method with invalid regex
|
||||
@ Scrapy is awsome
|
||||
"""
|
||||
pass
|
||||
|
||||
def invalid_regex_with_valid_contract(self, response):
|
||||
"""method with invalid regex
|
||||
@ scrapy is awsome
|
||||
@url http://scrapy.org
|
||||
"""
|
||||
pass
|
||||
|
||||
def returns_request_meta(self, response):
|
||||
"""method which returns request
|
||||
|
|
@ -235,7 +231,6 @@ class CustomContractSuccessSpider(Spider):
|
|||
"""
|
||||
@custom_success_contract
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CustomContractFailSpider(Spider):
|
||||
|
|
@ -245,7 +240,6 @@ class CustomContractFailSpider(Spider):
|
|||
"""
|
||||
@custom_fail_contract
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class InheritsTestSpider(TestSpider):
|
||||
|
|
|
|||
|
|
@ -894,7 +894,7 @@ class S3TestCase(unittest.TestCase):
|
|||
except Exception as e:
|
||||
self.assertIsInstance(e, (TypeError, NotConfigured))
|
||||
else:
|
||||
raise AssertionError()
|
||||
raise AssertionError
|
||||
|
||||
def test_request_signing1(self):
|
||||
# gets an object from the johnsmith bucket.
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
|
|||
|
||||
class InvalidProcessExceptionMiddleware:
|
||||
def process_request(self, request, spider):
|
||||
raise Exception()
|
||||
raise Exception
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
return 1
|
||||
|
|
@ -250,8 +250,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
|
|||
class CoroMiddleware:
|
||||
async def process_request(self, request, spider):
|
||||
await asyncio.sleep(0.1)
|
||||
result = await get_from_asyncio_queue(resp)
|
||||
return result
|
||||
return await get_from_asyncio_queue(resp)
|
||||
|
||||
self.mwman._add_middleware(CoroMiddleware())
|
||||
req = Request("http://example.com/index.html")
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ class HttpCompressionTest(TestCase):
|
|||
|
||||
def _getresponse(self, coding):
|
||||
if coding not in FORMAT:
|
||||
raise ValueError()
|
||||
raise ValueError
|
||||
|
||||
samplefile, contentencoding = FORMAT[coding]
|
||||
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ class MaxRetryTimesTest(unittest.TestCase):
|
|||
spider = spider or self.spider
|
||||
middleware = middleware or self.mw
|
||||
|
||||
for i in range(0, max_retry_times):
|
||||
for i in range(max_retry_times):
|
||||
req = middleware.process_exception(req, exception, spider)
|
||||
assert isinstance(req, Request)
|
||||
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ Disallow: /some/randome/page.html
|
|||
def test_robotstxt_garbage(self):
|
||||
# garbage response should be discarded, equal 'allow all'
|
||||
middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
|
||||
deferred = DeferredList(
|
||||
return DeferredList(
|
||||
[
|
||||
self.assertNotIgnored(Request("http://site.local"), middleware),
|
||||
self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
|
||||
|
|
@ -127,7 +127,6 @@ Disallow: /some/randome/page.html
|
|||
],
|
||||
fireOnOneErrback=True,
|
||||
)
|
||||
return deferred
|
||||
|
||||
def _get_emptybody_crawler(self):
|
||||
crawler = self.crawler
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class TelnetExtensionTest(unittest.TestCase):
|
|||
console = TelnetConsole(crawler)
|
||||
|
||||
# This function has some side effects we don't need for this test
|
||||
console._get_telnet_vars = lambda: {}
|
||||
console._get_telnet_vars = dict
|
||||
|
||||
console.start_listening()
|
||||
protocol = console.protocol()
|
||||
|
|
|
|||
|
|
@ -134,8 +134,7 @@ class FTPFeedStorageTest(unittest.TestCase):
|
|||
name = "test_spider"
|
||||
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
spider = TestSpider.from_crawler(crawler)
|
||||
return spider
|
||||
return TestSpider.from_crawler(crawler)
|
||||
|
||||
def _store(self, uri, content, feed_options=None, settings=None):
|
||||
crawler = get_crawler(settings_dict=settings or {})
|
||||
|
|
@ -210,8 +209,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
|
|||
name = "test_spider"
|
||||
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
spider = TestSpider.from_crawler(crawler)
|
||||
return spider
|
||||
return TestSpider.from_crawler(crawler)
|
||||
|
||||
def test_default_temp_dir(self):
|
||||
b = BlockingFeedStorage()
|
||||
|
|
@ -1759,13 +1757,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
|
|||
crawler = get_crawler(spider_cls, settings)
|
||||
yield crawler.crawl()
|
||||
|
||||
for file_path, feed_options in FEEDS.items():
|
||||
for file_path in FEEDS:
|
||||
content[str(file_path)] = (
|
||||
Path(file_path).read_bytes() if Path(file_path).exists() else None
|
||||
)
|
||||
|
||||
finally:
|
||||
for file_path in FEEDS.keys():
|
||||
for file_path in FEEDS:
|
||||
if not Path(file_path).exists():
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -342,13 +342,11 @@ class BaseResponseTest(unittest.TestCase):
|
|||
|
||||
def _links_response(self):
|
||||
body = get_testdata("link_extractor", "linkextractor.html")
|
||||
resp = self.response_class("http://example.com/index", body=body)
|
||||
return resp
|
||||
return self.response_class("http://example.com/index", body=body)
|
||||
|
||||
def _links_response_no_href(self):
|
||||
body = get_testdata("link_extractor", "linkextractor_no_href.html")
|
||||
resp = self.response_class("http://example.com/index", body=body)
|
||||
return resp
|
||||
return self.response_class("http://example.com/index", body=body)
|
||||
|
||||
|
||||
class TextResponseTest(BaseResponseTest):
|
||||
|
|
|
|||
|
|
@ -311,11 +311,11 @@ class FilesPipelineTestCaseFieldsDataClass(
|
|||
class FilesPipelineTestAttrsItem:
|
||||
name = attr.ib(default="")
|
||||
# default fields
|
||||
file_urls: list[str] = attr.ib(default=lambda: [])
|
||||
files: list[dict[str, str]] = attr.ib(default=lambda: [])
|
||||
file_urls: list[str] = attr.ib(default=list)
|
||||
files: list[dict[str, str]] = attr.ib(default=list)
|
||||
# overridden fields
|
||||
custom_file_urls: list[str] = attr.ib(default=lambda: [])
|
||||
custom_files: list[dict[str, str]] = attr.ib(default=lambda: [])
|
||||
custom_file_urls: list[str] = attr.ib(default=list)
|
||||
custom_files: list[dict[str, str]] = attr.ib(default=list)
|
||||
|
||||
|
||||
class FilesPipelineTestCaseFieldsAttrsItem(
|
||||
|
|
|
|||
|
|
@ -295,11 +295,11 @@ class ImagesPipelineTestCaseFieldsDataClass(
|
|||
class ImagesPipelineTestAttrsItem:
|
||||
name = attr.ib(default="")
|
||||
# default fields
|
||||
image_urls: list[str] = attr.ib(default=lambda: [])
|
||||
images: list[dict[str, str]] = attr.ib(default=lambda: [])
|
||||
image_urls: list[str] = attr.ib(default=list)
|
||||
images: list[dict[str, str]] = attr.ib(default=list)
|
||||
# overridden fields
|
||||
custom_image_urls: list[str] = attr.ib(default=lambda: [])
|
||||
custom_images: list[dict[str, str]] = attr.ib(default=lambda: [])
|
||||
custom_image_urls: list[str] = attr.ib(default=list)
|
||||
custom_images: list[dict[str, str]] = attr.ib(default=list)
|
||||
|
||||
|
||||
class ImagesPipelineTestCaseFieldsAttrsItem(
|
||||
|
|
|
|||
|
|
@ -48,8 +48,7 @@ sys.exit(mitmdump())
|
|||
)
|
||||
line = self.proc.stdout.readline().decode("utf-8")
|
||||
host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
|
||||
address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
|
||||
return address
|
||||
return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
|
||||
|
||||
def stop(self):
|
||||
self.proc.kill()
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ class InjectArgumentsDownloaderMiddleware:
|
|||
def process_request(self, request, spider):
|
||||
if request.callback.__name__ == "parse_downloader_mw":
|
||||
request.cb_kwargs["from_process_request"] = True
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if request.callback.__name__ == "parse_downloader_mw":
|
||||
|
|
@ -39,7 +38,6 @@ class InjectArgumentsSpiderMiddleware:
|
|||
request = response.request
|
||||
if request.callback.__name__ == "parse_spider_mw":
|
||||
request.cb_kwargs["from_process_spider_input"] = True
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
for element in result:
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ class SignalCatcherSpider(Spider):
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = cls(crawler, *args, **kwargs)
|
||||
return spider
|
||||
return cls(crawler, *args, **kwargs)
|
||||
|
||||
def on_request_left(self, request, spider):
|
||||
self.caught_times += 1
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
|
@ -18,8 +18,13 @@ from scrapy.utils.misc import load_object
|
|||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
MockEngine = collections.namedtuple("MockEngine", ["downloader"])
|
||||
MockSlot = collections.namedtuple("MockSlot", ["active"])
|
||||
|
||||
class MockEngine(NamedTuple):
|
||||
downloader: MockDownloader
|
||||
|
||||
|
||||
class MockSlot(NamedTuple):
|
||||
active: list[Any]
|
||||
|
||||
|
||||
class MockDownloader:
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ class SpiderMiddlewareTestCase(TestCase):
|
|||
results = []
|
||||
dfd.addBoth(results.append)
|
||||
self._wait(dfd)
|
||||
ret = results[0]
|
||||
return ret
|
||||
return results[0]
|
||||
|
||||
|
||||
class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
|
||||
|
|
@ -79,7 +78,7 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
|
|||
|
||||
class RaiseExceptionProcessSpiderOutputMiddleware:
|
||||
def process_spider_output(self, response, result, spider):
|
||||
raise Exception()
|
||||
raise Exception
|
||||
|
||||
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
|
||||
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
|
||||
|
|
@ -290,10 +289,7 @@ class ProcessSpiderOutputNonIterableMiddleware:
|
|||
|
||||
class ProcessSpiderOutputCoroutineMiddleware:
|
||||
async def process_spider_output(self, response, result, spider):
|
||||
results = []
|
||||
for r in result:
|
||||
results.append(r)
|
||||
return results
|
||||
return result
|
||||
|
||||
|
||||
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ class LogExceptionMiddleware:
|
|||
spider.logger.info(
|
||||
"Middleware: %s exception caught", exception.__class__.__name__
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# ================================================================================
|
||||
|
|
@ -44,7 +43,7 @@ class RecoverySpider(Spider):
|
|||
yield {"test": 1}
|
||||
self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
|
||||
if not response.meta.get("dont_fail"):
|
||||
raise TabError()
|
||||
raise TabError
|
||||
|
||||
|
||||
class RecoveryAsyncGenSpider(RecoverySpider):
|
||||
|
|
@ -60,7 +59,7 @@ class RecoveryAsyncGenSpider(RecoverySpider):
|
|||
class FailProcessSpiderInputMiddleware:
|
||||
def process_spider_input(self, response, spider):
|
||||
spider.logger.info("Middleware: will raise IndexError")
|
||||
raise IndexError()
|
||||
raise IndexError
|
||||
|
||||
|
||||
class ProcessSpiderInputSpiderWithoutErrback(Spider):
|
||||
|
|
@ -110,14 +109,14 @@ class GeneratorCallbackSpider(Spider):
|
|||
def parse(self, response):
|
||||
yield {"test": 1}
|
||||
yield {"test": 2}
|
||||
raise ImportError()
|
||||
raise ImportError
|
||||
|
||||
|
||||
class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
|
||||
async def parse(self, response):
|
||||
yield {"test": 1}
|
||||
yield {"test": 2}
|
||||
raise ImportError()
|
||||
raise ImportError
|
||||
|
||||
|
||||
# ================================================================================
|
||||
|
|
@ -170,7 +169,6 @@ class _GeneratorDoNothingMiddleware:
|
|||
def process_spider_exception(self, response, exception, spider):
|
||||
method = f"{self.__class__.__name__}.process_spider_exception"
|
||||
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
|
||||
return None
|
||||
|
||||
|
||||
class GeneratorFailMiddleware:
|
||||
|
|
@ -178,7 +176,7 @@ class GeneratorFailMiddleware:
|
|||
for r in result:
|
||||
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
|
||||
yield r
|
||||
raise LookupError()
|
||||
raise LookupError
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
method = f"{self.__class__.__name__}.process_spider_exception"
|
||||
|
|
@ -240,7 +238,6 @@ class _NotGeneratorDoNothingMiddleware:
|
|||
def process_spider_exception(self, response, exception, spider):
|
||||
method = f"{self.__class__.__name__}.process_spider_exception"
|
||||
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
|
||||
return None
|
||||
|
||||
|
||||
class NotGeneratorFailMiddleware:
|
||||
|
|
@ -249,7 +246,7 @@ class NotGeneratorFailMiddleware:
|
|||
for r in result:
|
||||
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
|
||||
out.append(r)
|
||||
raise ReferenceError()
|
||||
raise ReferenceError
|
||||
return out
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class BaseQueueTestCase(unittest.TestCase):
|
|||
|
||||
class RequestQueueTestMixin:
|
||||
def queue(self):
|
||||
raise NotImplementedError()
|
||||
raise NotImplementedError
|
||||
|
||||
def test_one_element_with_peek(self):
|
||||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@ class AsyncgenUtilsTest(unittest.TestCase):
|
|||
@deferred_f_from_coro_f
|
||||
async def test_as_async_generator(self):
|
||||
ag = as_async_generator(range(42))
|
||||
results = []
|
||||
async for i in ag:
|
||||
results.append(i)
|
||||
results = [i async for i in ag]
|
||||
self.assertEqual(results, list(range(42)))
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
|
|
|
|||
|
|
@ -26,15 +26,14 @@ class XmliterBaseTestCase:
|
|||
"""
|
||||
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
attrs = []
|
||||
for x in self.xmliter(response, "product"):
|
||||
attrs.append(
|
||||
(
|
||||
x.attrib["id"],
|
||||
x.xpath("name/text()").getall(),
|
||||
x.xpath("./type/text()").getall(),
|
||||
)
|
||||
attrs = [
|
||||
(
|
||||
x.attrib["id"],
|
||||
x.xpath("name/text()").getall(),
|
||||
x.xpath("./type/text()").getall(),
|
||||
)
|
||||
for x in self.xmliter(response, "product")
|
||||
]
|
||||
|
||||
self.assertEqual(
|
||||
attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])]
|
||||
|
|
@ -99,15 +98,14 @@ class XmliterBaseTestCase:
|
|||
# Unicode body needs encoding information
|
||||
XmlResponse(url="http://example.com", body=body, encoding="utf-8"),
|
||||
):
|
||||
attrs = []
|
||||
for x in self.xmliter(r, "þingflokkur"):
|
||||
attrs.append(
|
||||
(
|
||||
x.attrib["id"],
|
||||
x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
|
||||
x.xpath("./tímabil/fyrstaþing/text()").getall(),
|
||||
)
|
||||
attrs = [
|
||||
(
|
||||
x.attrib["id"],
|
||||
x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
|
||||
x.xpath("./tímabil/fyrstaþing/text()").getall(),
|
||||
)
|
||||
for x in self.xmliter(r, "þingflokkur")
|
||||
]
|
||||
|
||||
self.assertEqual(
|
||||
attrs,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from scrapy.utils.misc import (
|
|||
|
||||
|
||||
def _indentation_error(*args, **kwargs):
|
||||
raise IndentationError()
|
||||
raise IndentationError
|
||||
|
||||
|
||||
def top_level_return_something():
|
||||
|
|
|
|||
|
|
@ -58,13 +58,6 @@ class MutableAsyncChainTest(unittest.TestCase):
|
|||
for i in range(5, 7):
|
||||
yield i
|
||||
|
||||
@staticmethod
|
||||
async def collect_asyncgen_exc(asyncgen):
|
||||
results = []
|
||||
async for x in asyncgen:
|
||||
results.append(x)
|
||||
return results
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_mutableasyncchain(self):
|
||||
m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
|
||||
|
|
|
|||
Loading…
Reference in New Issue