mirror of https://github.com/scrapy/scrapy.git
Fix text replacement artifacts
This commit is contained in:
parent
cc49ea9f18
commit
b37f3b5422
|
|
@ -305,10 +305,10 @@ with multiples lines
|
|||
# basic asserts in case of weird communication errors
|
||||
assert "responses" in crawler.spider.meta
|
||||
assert "failures" not in crawler.spider.meta
|
||||
# test_start doesn't set Referer header
|
||||
# start() doesn't set Referer header
|
||||
echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
|
||||
assert "Referer" not in echo0["headers"]
|
||||
# following request sets Referer to test_start url
|
||||
# following request sets Referer to the source request url
|
||||
echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body))
|
||||
assert echo1["headers"].get("Referer") == [req0.url]
|
||||
# next request avoids Referer header
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware:
|
|||
Make sure spider middlewares are able to update the keyword arguments
|
||||
"""
|
||||
|
||||
def process_test_start(self, test_start, spider):
|
||||
for request in test_start:
|
||||
async def process_start(self, start):
|
||||
async for request in start:
|
||||
if request.callback.__name__ == "parse_spider_mw":
|
||||
request.cb_kwargs["from_process_test_start"] = True
|
||||
request.cb_kwargs["from_process_start"] = True
|
||||
yield request
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
|
|
@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
self.checks.append(bool(from_process_response))
|
||||
self.crawler.stats.inc_value("boolean_checks", 2)
|
||||
|
||||
def parse_spider_mw(
|
||||
self, response, from_process_spider_input, from_process_test_start
|
||||
):
|
||||
def parse_spider_mw(self, response, from_process_spider_input, from_process_start):
|
||||
self.checks.append(bool(from_process_spider_input))
|
||||
self.checks.append(bool(from_process_test_start))
|
||||
self.checks.append(bool(from_process_start))
|
||||
self.crawler.stats.inc_value("boolean_checks", 2)
|
||||
return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)
|
||||
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
|
|||
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
|
||||
"""Helpers for testing sync, async and mixed middlewares.
|
||||
|
||||
Should work for process_spider_output and, when it's supported, process_test_start.
|
||||
Should work for process_spider_output and, when it's supported, process_start.
|
||||
"""
|
||||
|
||||
ITEM_TYPE: type | tuple
|
||||
|
|
@ -321,8 +321,9 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
|
|||
|
||||
|
||||
class ProcessStartSimpleMiddleware:
|
||||
def process_test_start(self, test_start, spider):
|
||||
yield from test_start
|
||||
async def process_start(self, start):
|
||||
async for item_or_request in start:
|
||||
yield item_or_request
|
||||
|
||||
|
||||
class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
|
||||
|
|
|
|||
Loading…
Reference in New Issue