Fix text replacement artifacts

This commit is contained in:
Adrián Chaves 2025-03-24 11:22:41 +01:00
parent cc49ea9f18
commit b37f3b5422
3 changed files with 11 additions and 12 deletions

View File

@ -305,10 +305,10 @@ with multiples lines
# basic asserts in case of weird communication errors
assert "responses" in crawler.spider.meta
assert "failures" not in crawler.spider.meta
# test_start doesn't set Referer header
# start() doesn't set Referer header
echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
assert "Referer" not in echo0["headers"]
# following request sets Referer to test_start url
# following request sets Referer to the source request url
echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body))
assert echo1["headers"].get("Referer") == [req0.url]
# next request avoids Referer header

View File

@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware:
Make sure spider middlewares are able to update the keyword arguments
"""
def process_test_start(self, test_start, spider):
for request in test_start:
async def process_start(self, start):
async for request in start:
if request.callback.__name__ == "parse_spider_mw":
request.cb_kwargs["from_process_test_start"] = True
request.cb_kwargs["from_process_start"] = True
yield request
def process_spider_input(self, response, spider):
@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider):
self.checks.append(bool(from_process_response))
self.crawler.stats.inc_value("boolean_checks", 2)
def parse_spider_mw(
self, response, from_process_spider_input, from_process_test_start
):
def parse_spider_mw(self, response, from_process_spider_input, from_process_start):
self.checks.append(bool(from_process_spider_input))
self.checks.append(bool(from_process_test_start))
self.checks.append(bool(from_process_start))
self.crawler.stats.inc_value("boolean_checks", 2)
return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)

View File

@ -112,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
"""Helpers for testing sync, async and mixed middlewares.
Should work for process_spider_output and, when it's supported, process_test_start.
Should work for process_spider_output and, when it's supported, process_start.
"""
ITEM_TYPE: type | tuple
@ -321,8 +321,9 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
class ProcessStartSimpleMiddleware:
def process_test_start(self, test_start, spider):
yield from test_start
async def process_start(self, start):
async for item_or_request in start:
yield item_or_request
class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):