From b37f3b5422fbca2b8949b1e48dace50aeadc3f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 24 Mar 2025 11:22:41 +0100 Subject: [PATCH] Fix text replacement artifacts --- tests/test_crawl.py | 4 ++-- tests/test_request_cb_kwargs.py | 12 +++++------- tests/test_spidermiddleware.py | 7 ++++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 92b6238d7..9115edd27 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -305,10 +305,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # test_start doesn't set Referer header + # start() doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to test_start url + # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 21b5675fd..79b53b33b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_test_start(self, test_start, spider): - for request in test_start: + async def process_start(self, start): + async for request in start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_test_start"] = True + request.cb_kwargs["from_process_start"] = True yield request def process_spider_input(self, response, spider): @@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(bool(from_process_response)) self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw( - self, response, from_process_spider_input, from_process_test_start - ): + def parse_spider_mw(self, response, from_process_spider_input, from_process_start): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_test_start)) + self.checks.append(bool(from_process_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 3d8c885d1..9728d0d81 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_test_start. + Should work for process_spider_output and, when it's supported, process_start. """ ITEM_TYPE: type | tuple @@ -321,8 +321,9 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): class ProcessStartSimpleMiddleware: - def process_test_start(self, test_start, spider): - yield from test_start + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):