diff --git a/pyproject.toml b/pyproject.toml index 973d43162..b63a3631b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,8 @@ extend-select = [ "ISC", # flake8-logging "LOG", + # Perflint + "PERF", # pygrep-hooks "PGH", # flake8-pie @@ -314,6 +316,8 @@ ignore = [ "D402", # First word of the first line should be properly capitalized "D403", + # `try`-`except` within a loop incurs performance overhead + "PERF203", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 905959c25..237bd8331 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -7,10 +7,7 @@ _T = TypeVar("_T") async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: - results = [] - async for x in result: - results.append(x) - return results + return [x async for x in result] async def as_async_generator( diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 511511301..e954b625c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -235,8 +235,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str continue args.append(name) else: - for name in sig.parameters.keys(): - args.append(name) + args = list(sig.parameters) if stripself and args and args[0] == "self": args = args[1:] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 031d6180d..2debbe0d7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1757,13 +1757,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): crawler = get_crawler(spider_cls, settings) yield crawler.crawl() - for file_path, feed_options in FEEDS.items(): + for file_path in FEEDS: content[str(file_path)] = ( Path(file_path).read_bytes() if Path(file_path).exists() else None ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index af3b7543d..f2a57bd88 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -289,10 +289,7 @@ class ProcessSpiderOutputNonIterableMiddleware: class ProcessSpiderOutputCoroutineMiddleware: async def process_spider_output(self, response, result, spider): - results = [] - for r in result: - results.append(r) - return results + return result class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 9ae66c57c..8adeea5c0 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -8,9 +8,7 @@ class AsyncgenUtilsTest(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator(self): ag = as_async_generator(range(42)) - results = [] - async for i in ag: - results.append(i) + results = [i async for i in ag] self.assertEqual(results, list(range(42))) @deferred_f_from_coro_f diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ec377bb19..4c81e3a2f 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -26,15 +26,14 @@ class XmliterBaseTestCase: """ response = XmlResponse(url="http://example.com", body=body) - attrs = [] - for x in self.xmliter(response, "product"): - attrs.append( - ( - x.attrib["id"], - x.xpath("name/text()").getall(), - x.xpath("./type/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall(), ) + for x in self.xmliter(response, "product") + ] self.assertEqual( attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] @@ -99,15 +98,14 @@ class XmliterBaseTestCase: # Unicode body needs encoding information XmlResponse(url="http://example.com", body=body, encoding="utf-8"), ): - attrs = [] - for x in self.xmliter(r, "þingflokkur"): - attrs.append( - ( - x.attrib["id"], - x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), - x.xpath("./tímabil/fyrstaþing/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), + x.xpath("./tímabil/fyrstaþing/text()").getall(), ) + for x in self.xmliter(r, "þingflokkur") + ] self.assertEqual( attrs, diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index f80f2517a..83004cec4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -58,13 +58,6 @@ class MutableAsyncChainTest(unittest.TestCase): for i in range(5, 7): yield i - @staticmethod - async def collect_asyncgen_exc(asyncgen): - results = [] - async for x in asyncgen: - results.append(x) - return results - @deferred_f_from_coro_f async def test_mutableasyncchain(self): m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))