diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b273e269b..39b9a33aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,7 @@ repos: rev: v0.8.1 hooks: - id: ruff + args: [ --fix ] - repo: https://github.com/psf/black.git rev: 24.4.2 hooks: diff --git a/pyproject.toml b/pyproject.toml index 977792178..c0297e192 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,10 +240,20 @@ extend-select = [ "ISC", # flake8-logging "LOG", + # Perflint + "PERF", # pygrep-hooks "PGH", + # flake8-pie + "PIE", + # flake8-pyi + "PYI", # flake8-quotes "Q", + # flake8-return + "RET", + # flake8-raise + "RSE", # flake8-bandit "S", # flake8-slots @@ -254,6 +264,8 @@ extend-select = [ "TC", # pyupgrade "UP", + # pycodestyle warnings + "W", # flake8-2020 "YTT", ] @@ -306,6 +318,8 @@ ignore = [ "D402", # First word of the first line should be properly capitalized "D403", + # `try`-`except` within a loop incurs performance overhead + "PERF203", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 0d71ab6c6..86d4cc41c 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -22,7 +22,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: - raise UsageError() + raise UsageError if len(args) > 1: raise UsageError( "running 'scrapy crawl' with more than one spider is not supported" diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 0e046cece..d153a5271 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -28,7 +28,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: - raise UsageError() + raise UsageError editor = self.settings["EDITOR"] assert self.crawler_process diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 05e5e53e9..8a8d04ff6 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -68,7 +68,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): - raise UsageError() + raise UsageError request = Request( args[0], callback=self._print_response, diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 38f917c7e..d7dc104c2 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -101,7 +101,7 @@ class Command(ScrapyCommand): print(template_file.read_text(encoding="utf-8")) return if len(args) != 2: - raise UsageError() + raise UsageError name, url = args[0:2] url = verify_url_scheme(url) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fc16e46d1..cc5c1350b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -225,8 +225,9 @@ class Command(BaseRunSpiderCommand): cb_kwargs: dict[str, Any] | None = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} - d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) - return d + return maybeDeferred( + self.iterate_spider_output, callback(response, **cb_kwargs) + ) def get_callback_from_rules( self, spider: Spider, response: Response @@ -398,7 +399,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): - raise UsageError() + raise UsageError url = args[0] # prepare spidercls diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 55211f8d7..bf8e41020 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -43,7 +43,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: - raise UsageError() + raise UsageError filename = Path(args[0]) if not filename.exists(): raise UsageError(f"File not found: {filename}\n") diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 4ca015f5e..3047ae396 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -61,7 +61,6 @@ class Command(ScrapyCommand): """You can use this function to update the Scrapy objects that will be available in the shell """ - pass def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 6da877610..5cb73f0d2 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -92,7 +92,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): - raise UsageError() + raise UsageError project_name = args[0] diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f09d1903c..fcc94879a 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -81,7 +81,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param spider: the spider object for the current crawl :type spider: :class:`~scrapy.spiders.Spider` """ - pass def close(self, reason: str) -> Deferred[None] | None: """ @@ -91,14 +90,13 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param reason: a string which describes the reason why the spider was closed :type reason: :class:`str` """ - pass @abstractmethod def has_pending_requests(self) -> bool: """ ``True`` if the scheduler has enqueued requests, ``False`` otherwise """ - raise NotImplementedError() + raise NotImplementedError @abstractmethod def enqueue_request(self, request: Request) -> bool: @@ -112,7 +110,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): For reference, the default Scrapy scheduler returns ``False`` when the request is rejected by the dupefilter. """ - raise NotImplementedError() + raise NotImplementedError @abstractmethod def next_request(self) -> Request | None: @@ -124,7 +122,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): to the downloader in the current reactor cycle. The engine will continue calling ``next_request`` until ``has_pending_requests`` is ``False``. """ - raise NotImplementedError() + raise NotImplementedError class Scheduler(BaseScheduler): diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 05ec4cad4..a69f531a7 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -41,7 +41,7 @@ class OffsiteMiddleware: def process_request(self, request: Request, spider: Spider) -> None: if request.dont_filter or self.should_follow(request, spider): - return None + return domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index ea9f47d69..9411cff14 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING from twisted.internet.defer import Deferred, maybeDeferred @@ -31,8 +31,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_T = TypeVar("_T") - class RobotsTxtMiddleware: DOWNLOAD_PRIORITY: int = 1000 diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 7b8eea135..caf69daf4 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -50,7 +50,6 @@ class BaseDupeFilter: def log(self, request: Request, spider: Spider) -> None: """Log that a request has been filtered""" - pass class RFPDupeFilter(BaseDupeFilter): diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index e7ecdbe0c..96566ba86 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -13,8 +13,6 @@ from typing import Any class NotConfigured(Exception): """Indicates a missing configuration situation""" - pass - class _InvalidOutput(TypeError): """ @@ -22,8 +20,6 @@ class _InvalidOutput(TypeError): Internal and undocumented, it should not be raised or caught by user code. """ - pass - # HTTP and crawling @@ -35,8 +31,6 @@ class IgnoreRequest(Exception): class DontCloseSpider(Exception): """Request the spider not to be closed yet""" - pass - class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -64,14 +58,10 @@ class StopDownload(Exception): class DropItem(Exception): """Drop item from the item pipeline""" - pass - class NotSupported(Exception): """Indicates a feature or method is not supported""" - pass - # Commands @@ -89,10 +79,6 @@ class ScrapyDeprecationWarning(Warning): DeprecationWarning is silenced on Python 2.7+ """ - pass - class ContractFail(AssertionError): """Error raised in case of a failing contract""" - - pass diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index edea7cc39..b6e6f55a6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -586,7 +586,7 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - slot = FeedSlot( + return FeedSlot( storage=storage, uri=uri, format=feed_options["format"], @@ -600,7 +600,6 @@ class FeedExporter: settings=self.settings, crawler=self.crawler, ) - return slot def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 965d6434b..929807de8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -282,8 +282,7 @@ class DbmCacheStorage: headers = Headers(data["headers"]) body = data["body"] respcls = responsetypes.from_args(headers=headers, url=url, body=body) - response = respcls(url=url, headers=headers, status=status, body=body) - return response + return respcls(url=url, headers=headers, status=status, body=body) def store_response( self, spider: Spider, request: Request, response: Response @@ -349,8 +348,7 @@ class FilesystemCacheStorage: status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - response = respcls(url=url, headers=headers, status=status, body=body) - return response + return respcls(url=url, headers=headers, status=status, body=body) def store_response( self, spider: Spider, request: Request, response: Response diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 16067f82b..b1fa160c8 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -157,8 +157,7 @@ class PostProcessingManager(IOBase): return True def _load_plugins(self, plugins: list[Any]) -> list[Any]: - plugins = [load_object(plugin) for plugin in plugins] - return plugins + return [load_object(plugin) for plugin in plugins] def _get_head_plugin(self) -> Any: prev = self.file diff --git a/scrapy/link.py b/scrapy/link.py index 4bdbc1823..1a569f892 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -5,8 +5,6 @@ For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -from typing import Any - class Link: """Link objects represent an extracted link by the LinkExtractor. @@ -39,7 +37,7 @@ class Link: self.fragment: str = fragment self.nofollow: bool = nofollow - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: if not isinstance(other, Link): raise NotImplementedError return ( diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index bd96ccf19..f195dbdd7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -253,8 +253,7 @@ class LxmlLinkExtractor: if self.canonicalize: for link in links: link.url = canonicalize_url(link.url) - links = self.link_extractor._process_links(links) - return links + return self.link_extractor._process_links(links) def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]: return self.link_extractor._extract_links(*args, **kwargs) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 691a1cbf2..5438b8522 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -5,16 +5,7 @@ import logging import warnings from abc import ABC, abstractmethod from collections import defaultdict -from typing import ( - TYPE_CHECKING, - Any, - Literal, - NoReturn, - TypedDict, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Literal, NoReturn, TypedDict, Union, cast from twisted import version as twisted_version from twisted.internet.defer import Deferred, DeferredList @@ -41,8 +32,6 @@ if TYPE_CHECKING: from scrapy.http import Response from scrapy.utils.request import RequestFingerprinter -_T = TypeVar("_T") - class FileInfo(TypedDict): url: str @@ -293,12 +282,12 @@ class MediaPipeline(ABC): self, request: Request, info: SpiderInfo, *, item: Any = None ) -> Deferred[FileInfo | None]: """Check request before starting download""" - raise NotImplementedError() + raise NotImplementedError @abstractmethod def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]: """Returns the media requests to download""" - raise NotImplementedError() + raise NotImplementedError @abstractmethod def media_downloaded( @@ -310,14 +299,14 @@ class MediaPipeline(ABC): item: Any = None, ) -> FileInfo: """Handler for success downloads""" - raise NotImplementedError() + raise NotImplementedError @abstractmethod def media_failed( self, failure: Failure, request: Request, info: SpiderInfo ) -> NoReturn: """Handler for failed downloads""" - raise NotImplementedError() + raise NotImplementedError def item_completed( self, results: list[FileInfoOrError], item: Any, info: SpiderInfo @@ -345,4 +334,4 @@ class MediaPipeline(ABC): item: Any = None, ) -> str: """Returns the path where downloaded media should be stored""" - raise NotImplementedError() + raise NotImplementedError diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0e8260736..f5f00ab0f 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -76,7 +76,7 @@ class HostResolution: self.name: str = name def cancel(self) -> None: - raise NotImplementedError() + raise NotImplementedError @provider(IResolutionReceiver) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index f0a6e7467..417c9c142 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -52,7 +52,6 @@ class RobotParser(metaclass=ABCMeta): :param robotstxt_body: content of a robots.txt_ file. :type robotstxt_body: bytes """ - pass @abstractmethod def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: @@ -64,7 +63,6 @@ class RobotParser(metaclass=ABCMeta): :param user_agent: User agent :type user_agent: str or bytes """ - pass class PythonRobotParser(RobotParser): @@ -79,8 +77,7 @@ class PythonRobotParser(RobotParser): @classmethod def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) @@ -100,8 +97,7 @@ class RerpRobotParser(RobotParser): @classmethod def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) @@ -120,8 +116,7 @@ class ProtegoRobotParser(RobotParser): @classmethod def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 720217c97..93b7fcf17 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -51,7 +51,7 @@ class ReferrerPolicy: name: str def referrer(self, response_url: str, request_url: str) -> str | None: - raise NotImplementedError() + raise NotImplementedError def stripped_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 905959c25..237bd8331 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -7,10 +7,7 @@ _T = TypeVar("_T") async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: - results = [] - async for x in result: - results.append(x) - return results + return [x async for x in result] async def as_async_generator( diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 511511301..e954b625c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -235,8 +235,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str continue args.append(name) else: - for name in sig.parameters.keys(): - args.append(name) + args = list(sig.parameters) if stripself and args and args[0] == "self": args = args[1:] diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 2102ce798..2d781cc27 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -36,7 +36,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: return reactor.listenTCP(0, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) - for x in range(portrange[0], portrange[1] + 1): + for x in range(portrange[0], portrange[1] + 1): # noqa: RET503 try: return reactor.listenTCP(x, factory, interface=host) except error.CannotListenError: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a7ad4544d..76a6b7de6 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -53,7 +53,7 @@ def get_meta_refresh( return _metaref_cache[response] -def response_status_message(status: bytes | float | int | str) -> str: +def response_status_message(status: bytes | float | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, "Unknown Status") diff --git a/tests/spiders.py b/tests/spiders.py index 63c7a6f9b..3c44d7da5 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -175,7 +175,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: - return + return None reqs = [] for i in range(1, 3): req = Request(self.start_urls[0], dont_filter=True, meta={"req_id": i}) @@ -393,8 +393,8 @@ class DuplicateStartRequestsSpider(MockServerSpider): dupe_factor = 3 def start_requests(self): - for i in range(0, self.distinct_urls): - for j in range(0, self.dupe_factor): + for i in range(self.distinct_urls): + for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) diff --git a/tests/test_addons.py b/tests/test_addons.py index 775f629b3..17949997c 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -64,7 +64,7 @@ class AddonManagerTest(unittest.TestCase): def test_notconfigured(self): class NotConfiguredAddon: def update_settings(self, settings): - raise NotConfigured() + raise NotConfigured settings_dict = { "ADDONS": {NotConfiguredAddon: 0}, diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b0cb92d12..743889234 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -178,27 +178,23 @@ class TestSpider(Spider): """method with no url @returns items 1 1 """ - pass def custom_form(self, response): """ @url http://scrapy.org @custom_form """ - pass def invalid_regex(self, response): """method with invalid regex @ Scrapy is awsome """ - pass def invalid_regex_with_valid_contract(self, response): """method with invalid regex @ scrapy is awsome @url http://scrapy.org """ - pass def returns_request_meta(self, response): """method which returns request @@ -235,7 +231,6 @@ class CustomContractSuccessSpider(Spider): """ @custom_success_contract """ - pass class CustomContractFailSpider(Spider): @@ -245,7 +240,6 @@ class CustomContractFailSpider(Spider): """ @custom_fail_contract """ - pass class InheritsTestSpider(TestSpider): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6a7597e9f..3fcba4ef2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -894,7 +894,7 @@ class S3TestCase(unittest.TestCase): except Exception as e: self.assertIsInstance(e, (TypeError, NotConfigured)) else: - raise AssertionError() + raise AssertionError def test_request_signing1(self): # gets an object from the johnsmith bucket. diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index dd3f8ceb9..e650b4936 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -178,7 +178,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): - raise Exception() + raise Exception def process_exception(self, request, exception, spider): return 1 @@ -250,8 +250,7 @@ class MiddlewareUsingCoro(ManagerTestCase): class CoroMiddleware: async def process_request(self, request, spider): await asyncio.sleep(0.1) - result = await get_from_asyncio_queue(resp) - return result + return await get_from_asyncio_queue(resp) self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 934af6590..78d0dd99d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -59,7 +59,7 @@ class HttpCompressionTest(TestCase): def _getresponse(self, coding): if coding not in FORMAT: - raise ValueError() + raise ValueError samplefile, contentencoding = FORMAT[coding] diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index a010865ef..c99f19b03 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -265,7 +265,7 @@ class MaxRetryTimesTest(unittest.TestCase): spider = spider or self.spider middleware = middleware or self.mw - for i in range(0, max_retry_times): + for i in range(max_retry_times): req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 12b541456..535e07c1f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -116,7 +116,7 @@ Disallow: /some/randome/page.html def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - deferred = DeferredList( + return DeferredList( [ self.assertNotIgnored(Request("http://site.local"), middleware), self.assertNotIgnored(Request("http://site.local/allowed"), middleware), @@ -127,7 +127,6 @@ Disallow: /some/randome/page.html ], fireOnOneErrback=True, ) - return deferred def _get_emptybody_crawler(self): crawler = self.crawler diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 9fd680e9f..8c897c223 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -13,7 +13,7 @@ class TelnetExtensionTest(unittest.TestCase): console = TelnetConsole(crawler) # This function has some side effects we don't need for this test - console._get_telnet_vars = lambda: {} + console._get_telnet_vars = dict console.start_listening() protocol = console.protocol() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c3d429c2b..2debbe0d7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -134,8 +134,7 @@ class FTPFeedStorageTest(unittest.TestCase): name = "test_spider" crawler = get_crawler(settings_dict=settings) - spider = TestSpider.from_crawler(crawler) - return spider + return TestSpider.from_crawler(crawler) def _store(self, uri, content, feed_options=None, settings=None): crawler = get_crawler(settings_dict=settings or {}) @@ -210,8 +209,7 @@ class BlockingFeedStorageTest(unittest.TestCase): name = "test_spider" crawler = get_crawler(settings_dict=settings) - spider = TestSpider.from_crawler(crawler) - return spider + return TestSpider.from_crawler(crawler) def test_default_temp_dir(self): b = BlockingFeedStorage() @@ -1759,13 +1757,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): crawler = get_crawler(spider_cls, settings) yield crawler.crawl() - for file_path, feed_options in FEEDS.items(): + for file_path in FEEDS: content[str(file_path)] = ( Path(file_path).read_bytes() if Path(file_path).exists() else None ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b8a277295..679cc8238 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -342,13 +342,11 @@ class BaseResponseTest(unittest.TestCase): def _links_response(self): body = get_testdata("link_extractor", "linkextractor.html") - resp = self.response_class("http://example.com/index", body=body) - return resp + return self.response_class("http://example.com/index", body=body) def _links_response_no_href(self): body = get_testdata("link_extractor", "linkextractor_no_href.html") - resp = self.response_class("http://example.com/index", body=body) - return resp + return self.response_class("http://example.com/index", body=body) class TextResponseTest(BaseResponseTest): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 2be5e09bc..a6c5f0a94 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -311,11 +311,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: list[str] = attr.ib(default=lambda: []) - files: list[dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=list) + files: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_file_urls: list[str] = attr.ib(default=lambda: []) - custom_files: list[dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=list) + custom_files: list[dict[str, str]] = attr.ib(default=list) class FilesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3ffef4102..3d049843a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -295,11 +295,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: list[str] = attr.ib(default=lambda: []) - images: list[dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=list) + images: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_image_urls: list[str] = attr.ib(default=lambda: []) - custom_images: list[dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=list) + custom_images: list[dict[str, str]] = attr.ib(default=list) class ImagesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 93f006c76..26bd6332c 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -48,8 +48,7 @@ sys.exit(mitmdump()) ) line = self.proc.stdout.readline().decode("utf-8") host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) - address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" - return address + return f"http://{self.auth_user}:{self.auth_pass}@{host_port}" def stop(self): self.proc.kill() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8c0e5764a..b178c928b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -16,7 +16,6 @@ class InjectArgumentsDownloaderMiddleware: def process_request(self, request, spider): if request.callback.__name__ == "parse_downloader_mw": request.cb_kwargs["from_process_request"] = True - return None def process_response(self, request, response, spider): if request.callback.__name__ == "parse_downloader_mw": @@ -39,7 +38,6 @@ class InjectArgumentsSpiderMiddleware: request = response.request if request.callback.__name__ == "parse_spider_mw": request.cb_kwargs["from_process_spider_input"] = True - return None def process_spider_output(self, response, result, spider): for element in result: diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 54155f7ef..ba1b70695 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -18,8 +18,7 @@ class SignalCatcherSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = cls(crawler, *args, **kwargs) - return spider + return cls(crawler, *args, **kwargs) def on_request_left(self, request, spider): self.caught_times += 1 diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 387bc7c20..8bd1480ad 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,9 +1,9 @@ from __future__ import annotations -import collections import shutil import tempfile import unittest +from typing import Any, NamedTuple from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -18,8 +18,13 @@ from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -MockEngine = collections.namedtuple("MockEngine", ["downloader"]) -MockSlot = collections.namedtuple("MockSlot", ["active"]) + +class MockEngine(NamedTuple): + downloader: MockDownloader + + +class MockSlot(NamedTuple): + active: list[Any] class MockDownloader: diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 1a80eb7be..f2a57bd88 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -37,8 +37,7 @@ class SpiderMiddlewareTestCase(TestCase): results = [] dfd.addBoth(results.append) self._wait(dfd) - ret = results[0] - return ret + return results[0] class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): @@ -79,7 +78,7 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): class RaiseExceptionProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): - raise Exception() + raise Exception self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) @@ -290,10 +289,7 @@ class ProcessSpiderOutputNonIterableMiddleware: class ProcessSpiderOutputCoroutineMiddleware: async def process_spider_output(self, response, result, spider): - results = [] - for r in result: - results.append(r) - return results + return result class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index fad5dcaac..4c19d167f 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -12,7 +12,6 @@ class LogExceptionMiddleware: spider.logger.info( "Middleware: %s exception caught", exception.__class__.__name__ ) - return None # ================================================================================ @@ -44,7 +43,7 @@ class RecoverySpider(Spider): yield {"test": 1} self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail")) if not response.meta.get("dont_fail"): - raise TabError() + raise TabError class RecoveryAsyncGenSpider(RecoverySpider): @@ -60,7 +59,7 @@ class RecoveryAsyncGenSpider(RecoverySpider): class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): spider.logger.info("Middleware: will raise IndexError") - raise IndexError() + raise IndexError class ProcessSpiderInputSpiderWithoutErrback(Spider): @@ -110,14 +109,14 @@ class GeneratorCallbackSpider(Spider): def parse(self, response): yield {"test": 1} yield {"test": 2} - raise ImportError() + raise ImportError class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider): async def parse(self, response): yield {"test": 1} yield {"test": 2} - raise ImportError() + raise ImportError # ================================================================================ @@ -170,7 +169,6 @@ class _GeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return None class GeneratorFailMiddleware: @@ -178,7 +176,7 @@ class GeneratorFailMiddleware: for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - raise LookupError() + raise LookupError def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" @@ -240,7 +238,6 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return None class NotGeneratorFailMiddleware: @@ -249,7 +246,7 @@ class NotGeneratorFailMiddleware: for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) - raise ReferenceError() + raise ReferenceError return out def process_spider_exception(self, response, exception, spider): diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 499ca46b8..02ea8027f 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -41,7 +41,7 @@ class BaseQueueTestCase(unittest.TestCase): class RequestQueueTestMixin: def queue(self): - raise NotImplementedError() + raise NotImplementedError def test_one_element_with_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 9ae66c57c..8adeea5c0 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -8,9 +8,7 @@ class AsyncgenUtilsTest(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator(self): ag = as_async_generator(range(42)) - results = [] - async for i in ag: - results.append(i) + results = [i async for i in ag] self.assertEqual(results, list(range(42))) @deferred_f_from_coro_f diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ec377bb19..4c81e3a2f 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -26,15 +26,14 @@ class XmliterBaseTestCase: """ response = XmlResponse(url="http://example.com", body=body) - attrs = [] - for x in self.xmliter(response, "product"): - attrs.append( - ( - x.attrib["id"], - x.xpath("name/text()").getall(), - x.xpath("./type/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall(), ) + for x in self.xmliter(response, "product") + ] self.assertEqual( attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] @@ -99,15 +98,14 @@ class XmliterBaseTestCase: # Unicode body needs encoding information XmlResponse(url="http://example.com", body=body, encoding="utf-8"), ): - attrs = [] - for x in self.xmliter(r, "þingflokkur"): - attrs.append( - ( - x.attrib["id"], - x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), - x.xpath("./tímabil/fyrstaþing/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), + x.xpath("./tímabil/fyrstaþing/text()").getall(), ) + for x in self.xmliter(r, "þingflokkur") + ] self.assertEqual( attrs, diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 484757035..c7774751e 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -10,7 +10,7 @@ from scrapy.utils.misc import ( def _indentation_error(*args, **kwargs): - raise IndentationError() + raise IndentationError def top_level_return_something(): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index f80f2517a..83004cec4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -58,13 +58,6 @@ class MutableAsyncChainTest(unittest.TestCase): for i in range(5, 7): yield i - @staticmethod - async def collect_asyncgen_exc(asyncgen): - results = [] - async for x in asyncgen: - results.append(x) - return results - @deferred_f_from_coro_f async def test_mutableasyncchain(self): m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))