diff --git a/pyproject.toml b/pyproject.toml index b25ccb894..315857eda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -213,7 +213,6 @@ disable = [ "keyword-arg-before-vararg", "pointless-statement", "raise-missing-from", - "unbalanced-tuple-unpacking", "unnecessary-dunder-call", "used-before-assignment", ] @@ -234,10 +233,16 @@ filterwarnings = [ [tool.ruff.lint] extend-select = [ + # flake8-builtins + "A", + # flake8-async + "ASYNC", # flake8-bugbear "B", # flake8-comprehensions "C4", + # flake8-commas + "COM", # pydocstyle "D", # flake8-future-annotations @@ -296,6 +301,8 @@ extend-select = [ ignore = [ # Ones we want to ignore + # Trailing comma missing + "COM812", # Missing docstring in public module "D100", # Missing docstring in public class diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index eedaeb263..9ca383965 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -56,7 +56,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars: dict[str, Any]) -> None: + def update_vars(self, vars: dict[str, Any]) -> None: # noqa: A002 """You can use this function to update the Scrapy objects that will be available in the shell """ diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 50235846b..d8965c130 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,7 @@ from urllib.parse import urldefrag, urlparse from twisted.internet import ssl from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.internet.error import TimeoutError +from twisted.internet.error import TimeoutError as TxTimeoutError from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure from twisted.web.client import ( @@ -452,7 +452,7 @@ class ScrapyAgent: if self._txresponse: self._txresponse._transport.stopProducing() - raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") + raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: request.meta["download_latency"] = time() - start_time diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index d0a95ee9d..8e6236094 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,7 +4,7 @@ from time import time from typing import TYPE_CHECKING from urllib.parse import urldefrag -from twisted.internet.error import TimeoutError +from twisted.internet.error import TimeoutError as TxTimeoutError from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings @@ -127,4 +127,4 @@ class ScrapyH2Agent: return response url = urldefrag(request.url)[0] - raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") + raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 23335b7b2..cf2742de6 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -21,7 +21,7 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error -from twisted.internet.error import TimeoutError +from twisted.internet.error import TimeoutError as TxTimeoutError from twisted.internet.interfaces import ( IAddress, IHandshakeListener, @@ -322,7 +322,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._write_to_transport() self._lose_connection_with_error( - [TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")] + [TxTimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")] ) def connectionLost(self, reason: Failure = connectionDone) -> None: diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 3892dba23..c0d101698 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -8,11 +8,11 @@ from twisted.internet.error import ( ConnectError, ConnectionDone, ConnectionLost, - ConnectionRefusedError, DNSLookupError, TCPTimedOutError, - TimeoutError, ) +from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError +from twisted.internet.error import TimeoutError as TxTimeoutError from twisted.web.client import ResponseFailed from scrapy import signals @@ -34,9 +34,9 @@ if TYPE_CHECKING: class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( defer.TimeoutError, - TimeoutError, + TxTimeoutError, DNSLookupError, - ConnectionRefusedError, + TxConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0847c1e8b..983bbcfb9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -376,11 +376,11 @@ class FeedSlot: self, storage: FeedStorageProtocol, uri: str, - format: str, + format: str, # noqa: A002 store_empty: bool, batch_id: int, uri_template: str, - filter: ItemFilter, + filter: ItemFilter, # noqa: A002 feed_options: dict[str, Any], spider: Spider, exporters: dict[str, type[BaseItemExporter]], @@ -422,7 +422,7 @@ class FeedSlot: ) self.exporter = self._get_exporter( file=self.file, - format=self.feed_options["format"], + format_=self.feed_options["format"], fields_to_export=self.feed_options["fields"], encoding=self.feed_options["encoding"], indent=self.feed_options["indent"], @@ -436,10 +436,10 @@ class FeedSlot: self._exporting = True def _get_exporter( - self, file: IO[bytes], format: str, *args: Any, **kwargs: Any + self, file: IO[bytes], format_: str, *args: Any, **kwargs: Any ) -> BaseItemExporter: return build_from_crawler( - self.exporters[format], self.crawler, file, *args, **kwargs + self.exporters[format_], self.crawler, file, *args, **kwargs ) def finish_exporting(self) -> None: @@ -670,10 +670,10 @@ class FeedExporter: d[k] = load_object(v) return d - def _exporter_supported(self, format: str) -> bool: - if format in self.exporters: + def _exporter_supported(self, format_: str) -> bool: + if format_ in self.exporters: return True - logger.error("Unknown feed format: %(format)s", {"format": format}) + logger.error("Unknown feed format: %(format)s", {"format": format_}) return False def _settings_are_valid(self) -> bool: diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 01fd82830..5828ae52e 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -92,14 +92,14 @@ class LZMAPlugin: self.file = file self.feed_options = feed_options - format = self.feed_options.get("lzma_format") + format_ = self.feed_options.get("lzma_format") check = self.feed_options.get("lzma_check", -1) preset = self.feed_options.get("lzma_preset") filters = self.feed_options.get("lzma_filters") self.lzmafile = LZMAFile( filename=self.file, mode="wb", - format=format, + format=format_, check=check, preset=preset, filters=filters, diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index f8365a87b..99b22aca9 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -75,7 +75,7 @@ class Selector(_ParselSelector, object_ref): self, response: TextResponse | None = None, text: str | None = None, - type: str | None = None, + type: str | None = None, # noqa: A002 root: Any | None = _NOT_SET, **kwargs: Any, ): diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 8c5b843cb..9ea58c729 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -69,7 +69,7 @@ def is_asyncio_available() -> bool: async def _parallel_asyncio( iterable: Iterable[_T] | AsyncIterator[_T], count: int, - callable: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], + callable_: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], *args: _P.args, **kwargs: _P.kwargs, ) -> None: @@ -89,7 +89,7 @@ async def _parallel_asyncio( if item is None: break try: - await callable(item, *args, **kwargs) + await callable_(item, *args, **kwargs) finally: queue.task_done() diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 923ec005e..e34b7190f 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -27,8 +27,8 @@ class Root(Resource): return b"" -def _getarg(request, name: bytes, default: Any = None, type=str): - return type(request.args[name][0]) if name in request.args else default +def _getarg(request, name: bytes, default: Any = None, type_=str): + return type_(request.args[name][0]) if name in request.args else default if __name__ == "__main__": diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d161f30a6..fc149e185 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -157,7 +157,7 @@ def mustbe_deferred( def parallel( iterable: Iterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], _T2], # noqa: A002 *args: _P.args, **named: _P.kwargs, ) -> Deferred[list[tuple[bool, Iterator[_T2]]]]: @@ -220,12 +220,12 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, aiterable: AsyncIterator[_T], - callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], + callable_: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable + self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable_ self.callable_args: tuple[Any, ...] = callable_args self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False @@ -278,7 +278,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( async_iterable: AsyncIterator[_T], count: int, - callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], + callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], # noqa: A002 *args: _P.args, **named: _P.kwargs, ) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: @@ -295,7 +295,7 @@ def parallel_async( def process_chain( callbacks: Iterable[Callable[Concatenate[_T, _P], _T]], - input: _T, + input: _T, # noqa: A002 *a: _P.args, **kw: _P.kwargs, ) -> Deferred[_T]: @@ -310,7 +310,7 @@ def process_chain( def process_chain_both( callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], - input: Any, + input: Any, # noqa: A002 *a: _P.args, **kw: _P.kwargs, ) -> Deferred: @@ -334,7 +334,7 @@ def process_chain_both( def process_parallel( callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], - input: _T, + input: _T, # noqa: A002 *a: _P.args, **kw: _P.kwargs, ) -> Deferred[list[_T2]]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index decd4473c..1acb06757 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -241,7 +241,7 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: +def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noqa: A002 """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise @@ -279,7 +279,8 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: def warn_on_generator_with_return_value( - spider: Spider, callable: Callable[..., Any] + spider: Spider, + callable: Callable[..., Any], # noqa: A002 ) -> None: """ Logs a warning if a callable is a generator function and includes diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index a7f7f1356..e358eaca4 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -17,7 +17,7 @@ error = KeyError _DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB) -def open(file, flag="r", mode=0o666): +def open(file, flag="r", mode=0o666): # noqa: A001 """Open or create a dummy database compatible. Arguments ``flag`` and ``mode`` are ignored. diff --git a/tests/mockserver.py b/tests/mockserver.py index 841a2cfe4..7e53bc9dc 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -29,11 +29,11 @@ if TYPE_CHECKING: from twisted.internet.protocol import ServerFactory -def getarg(request, name, default=None, type=None): +def getarg(request, name, default=None, type_=None): if name in request.args: value = request.args[name][0] - if type is not None: - value = type(value) + if type_ is not None: + value = type_(value) return value return default @@ -129,11 +129,11 @@ class LeafResource(resource.Resource): class Follow(LeafResource): def render(self, request): - total = getarg(request, b"total", 100, type=int) - show = getarg(request, b"show", 1, type=int) + total = getarg(request, b"total", 100, type_=int) + show = getarg(request, b"show", 1, type_=int) order = getarg(request, b"order", b"desc") - maxlatency = getarg(request, b"maxlatency", 0, type=float) - n = getarg(request, b"n", total, type=int) + maxlatency = getarg(request, b"maxlatency", 0, type_=float) + n = getarg(request, b"n", total, type_=int) if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" @@ -157,8 +157,8 @@ class Follow(LeafResource): class Delay(LeafResource): def render_GET(self, request): - n = getarg(request, b"n", 1, type=float) - b = getarg(request, b"b", 1, type=int) + n = getarg(request, b"n", 1, type_=float) + b = getarg(request, b"b", 1, type_=int) if b: # send headers now and delay body request.write("") @@ -172,7 +172,7 @@ class Delay(LeafResource): class Status(LeafResource): def render_GET(self, request): - n = getarg(request, b"n", 200, type=int) + n = getarg(request, b"n", 200, type_=int) request.setResponseCode(n) return b"" @@ -229,7 +229,7 @@ class Partial(LeafResource): class Drop(Partial): def _delayedRender(self, request): - abort = getarg(request, b"abort", 0, type=int) + abort = getarg(request, b"abort", 0, type_=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport try: diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index ffdcdf49e..10fc88026 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -10,6 +10,8 @@ from twisted.internet.error import ( DNSLookupError, TCPTimedOutError, ) +from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError +from twisted.internet.error import TimeoutError as TxTimeoutError from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request @@ -91,12 +93,12 @@ class TestRetry: ConnectError, ConnectionDone, ConnectionLost, - ConnectionRefusedError, + TxConnectionRefusedError, defer.TimeoutError, DNSLookupError, ResponseFailed, TCPTimedOutError, - TimeoutError, + TxTimeoutError, ] for exc in exceptions: diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index d5e1b37f7..b38bf9570 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -16,12 +16,12 @@ from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider -def _get_dupefilter(*, crawler=None, settings=None, open=True): +def _get_dupefilter(*, crawler=None, settings=None, open_=True): if crawler is None: crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) dupefilter = scheduler.df - if open: + if open_: dupefilter.open() return dupefilter @@ -77,7 +77,7 @@ class TestRFPDupeFilter: path = tempfile.mkdtemp() try: - df = _get_dupefilter(settings={"JOBDIR": path}, open=False) + df = _get_dupefilter(settings={"JOBDIR": path}, open_=False) try: df.open() assert not df.request_seen(r1) @@ -85,7 +85,7 @@ class TestRFPDupeFilter: finally: df.close("finished") - df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False) + df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False) assert df != df2 try: df2.open() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index eb9081e7e..81c507ea1 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -20,7 +20,7 @@ from twisted.internet.defer import ( inlineCallbacks, ) from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint -from twisted.internet.error import TimeoutError +from twisted.internet.error import TimeoutError as TxTimeoutError from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS from twisted.trial.unittest import TestCase from twisted.web.client import URI, ResponseFailed @@ -632,7 +632,7 @@ class TestHttps2ClientProtocol(TestCase): for err in exc_info.value.reasons: from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415 - if isinstance(err, TimeoutError): + if isinstance(err, TxTimeoutError): assert ( f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" in str(err) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 74b4495ad..303b9596c 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -502,8 +502,8 @@ class TestImagesPipelineCustomSettings: assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value -def _create_image(format, *a, **kw): +def _create_image(format_, *a, **kw): buf = io.BytesIO() - Image.new(*a, **kw).save(buf, format) + Image.new(*a, **kw).save(buf, format_) buf.seek(0) return Image.open(buf), buf diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index b65f1b7e7..2050f8c83 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -159,7 +159,7 @@ class TestDownloaderAwarePriorityQueue: @pytest.mark.parametrize( - ("input", "output"), + ("input_", "output"), [ # By default, start requests are FIFO, other requests are LIFO. ([{}, {}], [2, 1]), @@ -172,7 +172,7 @@ class TestDownloaderAwarePriorityQueue: ([{"start": True}, {}], [2, 1]), ], ) -def test_pop_order(input, output): +def test_pop_order(input_, output): def make_url(index): return f"https://toscrape.com/{index}" @@ -187,7 +187,7 @@ def test_pop_order(input, output): ) input_requests = [ - make_request(index, data) for index, data in enumerate(input, start=1) + make_request(index, data) for index, data in enumerate(input_, start=1) ] expected_output_urls = [make_url(index) for index in output] diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 352e49165..c58b02ca5 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -353,26 +353,26 @@ class TestLocalWeakReferencedCache: assert len(cache) == 0 def test_cache_without_limit(self): - max = 10**4 + maximum = 10**4 cache = LocalWeakReferencedCache() refs = [] - for x in range(max): + for x in range(maximum): refs.append(Request(f"https://example.org/{x}")) cache[refs[-1]] = x - assert len(cache) == max + assert len(cache) == maximum for i, r in enumerate(refs): assert r in cache assert cache[r] == i del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache - for _ in range(max // 2): + for _ in range(maximum // 2): refs.pop() # PyPy takes longer to collect dead references garbage_collect() - assert len(cache) == max // 2 + assert len(cache) == maximum // 2 for i, r in enumerate(refs): assert r in cache assert cache[r] == i diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 3b1e626c9..8b02116b8 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -77,10 +77,10 @@ def test_pformat_no_pygments(isatty): real_import = builtins.__import__ - def mock_import(name, globals, locals, fromlist, level): + def mock_import(name, globals_, locals_, fromlist, level): if "pygments" in name: raise ImportError - return real_import(name, globals, locals, fromlist, level) + return real_import(name, globals_, locals_, fromlist, level) builtins.__import__ = mock_import assert pformat(value) == plain_string diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ac32fff2c..73e55b736 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -231,11 +231,11 @@ class TestXmliterBase(ABC): "onetwo" ) - iter = self.xmliter(body, "product") - next(iter) - next(iter) + my_iter = self.xmliter(body, "product") + next(my_iter) + next(my_iter) with pytest.raises(StopIteration): - next(iter) + next(my_iter) def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") @@ -473,13 +473,13 @@ class TestUtilsCsv: body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) - iter = csviter(response) - next(iter) - next(iter) - next(iter) - next(iter) + my_iter = csviter(response) + next(my_iter) + next(my_iter) + next(my_iter) + next(my_iter) with pytest.raises(StopIteration): - next(iter) + next(my_iter) def test_csviter_encoding(self): body1 = get_testdata("feeds", "feed-sample4.csv") diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index dcaac7358..2e6a790f8 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -30,7 +30,7 @@ class TestJsonEncoder: dt_set = {dt} dt_sets = [dts] - for input, output in [ + for input_, output in [ ("foo", "foo"), (d, ds), (t, ts), @@ -40,7 +40,7 @@ class TestJsonEncoder: (s, ss), (dt_set, dt_sets), ]: - assert encoder.encode(input) == json.dumps(output, sort_keys=True) + assert encoder.encode(input_) == json.dumps(output, sort_keys=True) def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None: assert "Deferred" in encoder.encode(defer.Deferred())