From 93644f2c30ee74a68584c9e4cdfd0827c2187d34 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 12 Dec 2024 20:27:04 +0500 Subject: [PATCH] Add flake8-pie rules to ruff. --- pyproject.toml | 2 ++ scrapy/commands/shell.py | 1 - scrapy/core/scheduler.py | 2 -- scrapy/dupefilters.py | 1 - scrapy/exceptions.py | 14 -------------- scrapy/robotstxt.py | 2 -- tests/spiders.py | 4 ++-- tests/test_contracts.py | 6 ------ tests/test_downloadermiddleware_retry.py | 2 +- tests/test_extension_telnet.py | 2 +- tests/test_pipeline_files.py | 8 ++++---- tests/test_pipeline_images.py | 8 ++++---- 12 files changed, 14 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 131684724..b3dd9f057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -242,6 +242,8 @@ extend-select = [ "LOG", # pygrep-hooks "PGH", + # flake8-pie + "PIE", # flake8-quotes "Q", # flake8-return diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 4ca015f5e..3047ae396 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -61,7 +61,6 @@ class Command(ScrapyCommand): """You can use this function to update the Scrapy objects that will be available in the shell """ - pass def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 823677175..fcc94879a 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -81,7 +81,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param spider: the spider object for the current crawl :type spider: :class:`~scrapy.spiders.Spider` """ - pass def close(self, reason: str) -> Deferred[None] | None: """ @@ -91,7 +90,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param reason: a string which describes the reason why the spider was closed :type reason: :class:`str` """ - pass @abstractmethod def has_pending_requests(self) -> bool: diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 7b8eea135..caf69daf4 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -50,7 +50,6 @@ class BaseDupeFilter: def log(self, request: Request, spider: Spider) -> None: """Log that a request has been filtered""" - pass class RFPDupeFilter(BaseDupeFilter): diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index e7ecdbe0c..96566ba86 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -13,8 +13,6 @@ from typing import Any class NotConfigured(Exception): """Indicates a missing configuration situation""" - pass - class _InvalidOutput(TypeError): """ @@ -22,8 +20,6 @@ class _InvalidOutput(TypeError): Internal and undocumented, it should not be raised or caught by user code. """ - pass - # HTTP and crawling @@ -35,8 +31,6 @@ class IgnoreRequest(Exception): class DontCloseSpider(Exception): """Request the spider not to be closed yet""" - pass - class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -64,14 +58,10 @@ class StopDownload(Exception): class DropItem(Exception): """Drop item from the item pipeline""" - pass - class NotSupported(Exception): """Indicates a feature or method is not supported""" - pass - # Commands @@ -89,10 +79,6 @@ class ScrapyDeprecationWarning(Warning): DeprecationWarning is silenced on Python 2.7+ """ - pass - class ContractFail(AssertionError): """Error raised in case of a failing contract""" - - pass diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 844969c6d..417c9c142 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -52,7 +52,6 @@ class RobotParser(metaclass=ABCMeta): :param robotstxt_body: content of a robots.txt_ file. :type robotstxt_body: bytes """ - pass @abstractmethod def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: @@ -64,7 +63,6 @@ class RobotParser(metaclass=ABCMeta): :param user_agent: User agent :type user_agent: str or bytes """ - pass class PythonRobotParser(RobotParser): diff --git a/tests/spiders.py b/tests/spiders.py index 0180cf757..3c44d7da5 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -393,8 +393,8 @@ class DuplicateStartRequestsSpider(MockServerSpider): dupe_factor = 3 def start_requests(self): - for i in range(0, self.distinct_urls): - for j in range(0, self.dupe_factor): + for i in range(self.distinct_urls): + for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b0cb92d12..743889234 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -178,27 +178,23 @@ class TestSpider(Spider): """method with no url @returns items 1 1 """ - pass def custom_form(self, response): """ @url http://scrapy.org @custom_form """ - pass def invalid_regex(self, response): """method with invalid regex @ Scrapy is awsome """ - pass def invalid_regex_with_valid_contract(self, response): """method with invalid regex @ scrapy is awsome @url http://scrapy.org """ - pass def returns_request_meta(self, response): """method which returns request @@ -235,7 +231,6 @@ class CustomContractSuccessSpider(Spider): """ @custom_success_contract """ - pass class CustomContractFailSpider(Spider): @@ -245,7 +240,6 @@ class CustomContractFailSpider(Spider): """ @custom_fail_contract """ - pass class InheritsTestSpider(TestSpider): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index a010865ef..c99f19b03 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -265,7 +265,7 @@ class MaxRetryTimesTest(unittest.TestCase): spider = spider or self.spider middleware = middleware or self.mw - for i in range(0, max_retry_times): + for i in range(max_retry_times): req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 9fd680e9f..8c897c223 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -13,7 +13,7 @@ class TelnetExtensionTest(unittest.TestCase): console = TelnetConsole(crawler) # This function has some side effects we don't need for this test - console._get_telnet_vars = lambda: {} + console._get_telnet_vars = dict console.start_listening() protocol = console.protocol() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 2be5e09bc..a6c5f0a94 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -311,11 +311,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: list[str] = attr.ib(default=lambda: []) - files: list[dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=list) + files: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_file_urls: list[str] = attr.ib(default=lambda: []) - custom_files: list[dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=list) + custom_files: list[dict[str, str]] = attr.ib(default=list) class FilesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3ffef4102..3d049843a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -295,11 +295,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: list[str] = attr.ib(default=lambda: []) - images: list[dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=list) + images: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_image_urls: list[str] = attr.ib(default=lambda: []) - custom_images: list[dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=list) + custom_images: list[dict[str, str]] = attr.ib(default=list) class ImagesPipelineTestCaseFieldsAttrsItem(