RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow 'callback' key in keyword arguments for request callbacks (#5251)

This commit is contained in:
Eugenio Lacuesta 2021-10-01 04:50:42 -03:00 committed by GitHub
parent 7d557f898c
commit 890f884de4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/core/scraper.py
View File

@ -156,7 +156,7 @@ class Scraper:
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallback(callback, **result.request.cb_kwargs)
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)

13
tests/test_request_cb_kwargs.py
View File

@ -60,7 +60,7 @@ class KeywordArgumentsSpider(MockServerSpider):
checks = []
def start_requests(self):
data = {'key': 'value', 'number': 123}
data = {'key': 'value', 'number': 123, 'callback': 'some_callback'}
yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
yield Request(self.mockserver.url('/general_without'), self.parse_general)
@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider):
if response.url.endswith('/general_with'):
self.checks.append(kwargs['key'] == 'value')
self.checks.append(kwargs['number'] == 123)
self.crawler.stats.inc_value('boolean_checks', 2)
self.checks.append(kwargs['callback'] == 'some_callback')
self.crawler.stats.inc_value('boolean_checks', 3)
elif response.url.endswith('/general_without'):
self.checks.append(kwargs == {})
self.crawler.stats.inc_value('boolean_checks')
@ -110,7 +111,7 @@ class KeywordArgumentsSpider(MockServerSpider):
TypeError: parse_takes_less() got an unexpected keyword argument 'number'
"""
def parse_takes_more(self, response, key, number, other):
def parse_takes_more(self, response, key, number, callback, other):
"""
Should raise
TypeError: parse_takes_more() missing 1 required positional argument: 'other'
@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
self.assertTrue(
str(exceptions['takes_less'].exc_info[1]).endswith(
"parse_takes_less() got an unexpected keyword argument 'number'"
)
),
msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]),
)
self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
self.assertTrue(
str(exceptions['takes_more'].exc_info[1]).endswith(
"parse_takes_more() missing 1 required positional argument: 'other'"
)
),
msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]),
)